You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2016/08/31 20:03:41 UTC

[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1109 - Failure

Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1109/

10 tests failed.
FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testResilienceWithDeleteByQueryOnTarget

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard1

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard1
	at __randomizedtesting.SeedInfo.seed([CAAE2B1DF4390888:6ACCB9872460CDCD]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testResilienceWithDeleteByQueryOnTarget(CdcrReplicationDistributedZkTest.java:588)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchBoundaries

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard2

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard2
	at __randomizedtesting.SeedInfo.seed([CAAE2B1DF4390888:E881F2488D9DC96F]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchBoundaries(CdcrReplicationDistributedZkTest.java:558)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable

Error Message:
Timeout while trying to assert replication errors

Stack Trace:
java.lang.AssertionError: Timeout while trying to assert replication errors
	at __randomizedtesting.SeedInfo.seed([CAAE2B1DF4390888:2C3A716EFF6665A0]:0)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable(CdcrReplicationDistributedZkTest.java:145)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.AssertionError
	at org.junit.Assert.fail(Assert.java:92)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertTrue(Assert.java:54)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable(CdcrReplicationDistributedZkTest.java:132)
	... 41 more


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:58377

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:58377
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:516)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:52014/_/v/c8n_1x3_lf_shard1_replica1]

Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: No live SolrServers available to handle this request:[http://127.0.0.1:52014/_/v/c8n_1x3_lf_shard1_replica1]
	at __randomizedtesting.SeedInfo.seed([CAAE2B1DF4390888:42FA14C75AC56570]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:755)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1161)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1050)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:992)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.sendDocsWithRetry(AbstractFullDistribZkTestBase.java:753)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.sendDocsWithRetry(AbstractFullDistribZkTestBase.java:741)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:178)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:57)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:52014/_/v/c8n_1x3_lf_shard1_replica1]
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:394)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:730)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more
Caused by: org.apache.solr.client.solrj.SolrServerException: Server refused connection at: http://127.0.0.1:52014/_/v/c8n_1x3_lf_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:615)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:415)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:383)
	... 6 more
Caused by: org.apache.http.conn.HttpHostConnectException: Connect to 127.0.0.1:52014 [/127.0.0.1] failed: Connection refused
	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:151)
	at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
	at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 10 more
Caused by: java.net.ConnectException: Connection refused
	at java.net.PlainSocketImpl.socketConnect(Native Method)
	at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
	at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
	at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
	at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
	at java.net.Socket.connect(Socket.java:589)
	at org.apache.http.conn.socket.PlainConnectionSocketFactory.connectSocket(PlainConnectionSocketFactory.java:74)
	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:134)
	... 20 more


FAILED:  org.apache.solr.cloud.ForceLeaderTest.testReplicasInLIRNoLeader

Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:42892/_/kx/forceleader_test_collection_shard1_replica2]

Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: No live SolrServers available to handle this request:[http://127.0.0.1:42892/_/kx/forceleader_test_collection_shard1_replica2]
	at __randomizedtesting.SeedInfo.seed([CAAE2B1DF4390888:2C391FDDCDBBF1E9]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:755)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1161)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1050)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:992)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.sendDocsWithRetry(AbstractFullDistribZkTestBase.java:753)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.sendDocsWithRetry(AbstractFullDistribZkTestBase.java:741)
	at org.apache.solr.cloud.ForceLeaderTest.sendDoc(ForceLeaderTest.java:424)
	at org.apache.solr.cloud.ForceLeaderTest.testReplicasInLIRNoLeader(ForceLeaderTest.java:131)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:42892/_/kx/forceleader_test_collection_shard1_replica2]
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:394)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:730)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more
Caused by: org.apache.solr.client.solrj.SolrServerException: Server refused connection at: http://127.0.0.1:42892/_/kx/forceleader_test_collection_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:615)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:415)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:383)
	... 6 more
Caused by: org.apache.http.conn.HttpHostConnectException: Connect to 127.0.0.1:42892 [/127.0.0.1] failed: Connection refused
	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:151)
	at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
	at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 10 more
Caused by: java.net.ConnectException: Connection refused
	at java.net.PlainSocketImpl.socketConnect(Native Method)
	at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
	at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
	at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
	at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
	at java.net.Socket.connect(Socket.java:589)
	at org.apache.http.conn.socket.PlainConnectionSocketFactory.connectSocket(PlainConnectionSocketFactory.java:74)
	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:134)
	... 20 more


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestStressReplication

Error Message:
timed out waiting for collection1 startAt time to exceed: Thu Sep 01 04:54:40 CHUT 2016

Stack Trace:
java.lang.AssertionError: timed out waiting for collection1 startAt time to exceed: Thu Sep 01 04:54:40 CHUT 2016
	at __randomizedtesting.SeedInfo.seed([CAAE2B1DF4390888:11052BDBF111613B]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.handler.TestReplicationHandler.watchCoreStartAt(TestReplicationHandler.java:1508)
	at org.apache.solr.handler.TestReplicationHandler.doTestStressReplication(TestReplicationHandler.java:858)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard1

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard1
	at __randomizedtesting.SeedInfo.seed([CAAE2B1DF4390888:96B3CC9C3A19BCBE]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart(CdcrReplicationDistributedZkTest.java:235)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
	at __randomizedtesting.SeedInfo.seed([CAAE2B1DF4390888]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest:     1) Thread[id=22062, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=22057, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=22082, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=22052, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=22067, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=22037, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=22072, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=22042, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=22047, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=22077, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 
   1) Thread[id=22062, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=22057, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=22082, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=22052, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=22067, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=22037, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=22072, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=22042, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=22047, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=22077, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([CAAE2B1DF4390888]:0)




Build Log:
[...truncated 11613 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_CAAE2B1DF4390888-001/init-core-data-001
   [junit4]   2> 51675 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[CAAE2B1DF4390888]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 51923 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[CAAE2B1DF4390888]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 51949 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 51961 INFO  (Thread-35) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 51961 INFO  (Thread-35) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 52132 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.ZkTestServer start zk server on port:44563
   [junit4]   2> 52133 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 52149 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 52160 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Exception causing close of session 0x0 due to java.io.IOException: ZooKeeperServer not running
   [junit4]   2> 53805 INFO  (zkCallback-32-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@7dc2bc1b name:ZooKeeperConnection Watcher:127.0.0.1:44563 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 53805 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 53805 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 53805 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 53813 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 53835 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 53839 INFO  (zkCallback-33-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@42eff37a name:ZooKeeperConnection Watcher:127.0.0.1:44563/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 53839 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 53840 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 53840 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1
   [junit4]   2> 53849 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/shards
   [junit4]   2> 53859 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection
   [junit4]   2> 53861 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection/shards
   [junit4]   2> 53862 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 53863 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 53867 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 53867 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/schema.xml
   [junit4]   2> 53869 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 53869 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 53885 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 53885 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 53888 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 53888 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/protwords.txt
   [junit4]   2> 53890 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 53890 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/currency.xml
   [junit4]   2> 53892 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 53892 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 53895 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 53895 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 53897 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 53897 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 53902 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 53902 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 53904 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 53904 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 53926 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 53960 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 53966 INFO  (zkCallback-35-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6852de2 name:ZooKeeperConnection Watcher:127.0.0.1:44563/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 53966 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 53966 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 53968 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x156e1d293e80002, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 54163 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_CAAE2B1DF4390888-001/control-001/cores/collection1
   [junit4]   2> 54247 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 54365 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4fa6eb77{/,null,AVAILABLE}
   [junit4]   2> 55137 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.e.j.s.ServerConnector Started ServerConnector@44aab337{SSL,[ssl, http/1.1]}{127.0.0.1:49385}
   [junit4]   2> 55137 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.e.j.s.Server Started @59695ms
   [junit4]   2> 55137 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_CAAE2B1DF4390888-001/tempDir-001/control/data, hostContext=/, hostPort=49385, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_CAAE2B1DF4390888-001/control-001/cores}
   [junit4]   2> 55140 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 55144 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_CAAE2B1DF4390888-001/control-001'
   [junit4]   2> 55144 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 55144 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 55146 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 55163 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 55187 INFO  (zkCallback-36-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@21fb6de1 name:ZooKeeperConnection Watcher:127.0.0.1:44563/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 55187 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 55187 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 55208 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 55209 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_CAAE2B1DF4390888-001/control-001/solr.xml
   [junit4]   2> 55221 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_CAAE2B1DF4390888-001/control-001/cores
   [junit4]   2> 55221 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.CoreContainer New CoreContainer 685098092
   [junit4]   2> 55221 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_CAAE2B1DF4390888-001/control-001]
   [junit4]   2> 55221 WARN  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.CoreContainer Couldn't add files from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_CAAE2B1DF4390888-001/control-001/lib to classpath: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_CAAE2B1DF4390888-001/control-001/lib
   [junit4]   2> 55221 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,
   [junit4]   2> 55265 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.u.UpdateShardHandler Created UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&maxConnections=100000&maxConnectionsPerHost=100000
   [junit4]   2> 55265 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 55265 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 55265 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44563/solr
   [junit4]   2> 55265 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 55266 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 55282 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 55300 INFO  (zkCallback-39-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@72b01c8a name:ZooKeeperConnection Watcher:127.0.0.1:44563 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 55302 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 55302 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 55310 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$74/1552792430@56cbae7e
   [junit4]   2> 55328 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 55356 INFO  (zkCallback-40-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@621bdbb7 name:ZooKeeperConnection Watcher:127.0.0.1:44563/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 55358 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 55360 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
   [junit4]   2> 55367 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
   [junit4]   2> 55370 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
   [junit4]   2> 55373 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
   [junit4]   2> 55375 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
   [junit4]   2> 55376 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 55386 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 55388 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 55395 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 55396 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 55397 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.ZkStateReader Loaded cluster properties: {urlScheme=https}
   [junit4]   2> 55397 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 55401 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
   [junit4]   2> 55402 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
   [junit4]   2> 55403 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 55404 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/96512736575881221-127.0.0.1:49385_-n_0000000000
   [junit4]   2> 55404 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:49385_
   [junit4]   2> 55404 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
   [junit4]   2> 55406 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.Overseer Overseer (id=96512736575881221-127.0.0.1:49385_-n_0000000000) starting
   [junit4]   2> 55408 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
   [junit4]   2> 55414 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=10000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 55425 INFO  (OverseerStateUpdate-96512736575881221-127.0.0.1:49385_-n_0000000000) [n:127.0.0.1:49385_    ] o.a.s.c.Overseer Starting to work on the main queue
   [junit4]   2> 55436 INFO  (OverseerStateUpdate-96512736575881221-127.0.0.1:49385_-n_0000000000) [n:127.0.0.1:49385_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 55444 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49385_
   [junit4]   2> 55444 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:49385_
   [junit4]   2> 55446 INFO  (zkCallback-40-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 55446 INFO  (OverseerCollectionConfigSetProcessor-96512736575881221-127.0.0.1:49385_-n_0000000000) [n:127.0.0.1:49385_    ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
   [junit4]   2> 55451 INFO  (zkCallback-40-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 55481 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 55481 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 55482 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_CAAE2B1DF4390888-001/control-001/cores
   [junit4]   2> 55483 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.json, collection=control_collection, coreNodeName=, dataDir=data/, shard=}
   [junit4]   2> 55484 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.CorePropertiesLocator Found core collection1 in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_CAAE2B1DF4390888-001/control-001/cores/collection1
   [junit4]   2> 55484 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions
   [junit4]   2> 55580 INFO  (coreLoadExecutor-105-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_ c:control_collection   x:collection1] o.a.s.c.ZkController publishing state=down
   [junit4]   2> 55580 INFO  (coreLoadExecutor-105-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_ c:control_collection   x:collection1] o.a.s.c.ZkController numShards not found on descriptor - reading it from system property
   [junit4]   2> 55582 INFO  (coreLoadExecutor-105-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_ c:control_collection   x:collection1] o.a.s.c.ZkController Collection control_collection not visible yet, but flagging it so a watch is registered when it becomes visible
   [junit4]   2> 55583 INFO  (OverseerStateUpdate-96512736575881221-127.0.0.1:49385_-n_0000000000) [n:127.0.0.1:49385_    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"https://127.0.0.1:49385",
   [junit4]   2>   "node_name":"127.0.0.1:49385_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":null,
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "operation":"state"} current state version: 0
   [junit4]   2> 55583 INFO  (OverseerStateUpdate-96512736575881221-127.0.0.1:49385_-n_0000000000) [n:127.0.0.1:49385_    ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"https://127.0.0.1:49385",
   [junit4]   2>   "node_name":"127.0.0.1:49385_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":null,
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 55584 INFO  (OverseerStateUpdate-96512736575881221-127.0.0.1:49385_-n_0000000000) [n:127.0.0.1:49385_    ] o.a.s.c.o.ClusterStateMutator building a new cName: control_collection
   [junit4]   2> 55585 INFO  (coreLoadExecutor-105-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_ c:control_collection   x:collection1] o.a.s.c.c.ZkStateReader Deleting data for [control_collection]
   [junit4]   2> 55586 INFO  (OverseerStateUpdate-96512736575881221-127.0.0.1:49385_-n_0000000000) [n:127.0.0.1:49385_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 55588 INFO  (zkCallback-40-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 55611 INFO  (coreLoadExecutor-105-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_ c:control_collection   x:collection1] o.a.s.c.ZkController look for our core node name
   [junit4]   2> 55611 INFO  (coreLoadExecutor-105-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_ c:control_collection   x:collection1] o.a.s.c.ZkController waiting to find shard id in clusterstate for collection1
   [junit4]   2> 55611 INFO  (coreLoadExecutor-105-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_ c:control_collection   x:collection1] o.a.s.c.ZkController Check for collection zkNode:control_collection
   [junit4]   2> 55611 INFO  (coreLoadExecutor-105-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_ c:control_collection   x:collection1] o.a.s.c.ZkController Collection zkNode exists
   [junit4]   2> 55611 INFO  (coreLoadExecutor-105-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_ c:control_collection   x:collection1] o.a.s.c.c.ZkStateReader Load collection config from: [/collections/control_collection]
   [junit4]   2> 55612 INFO  (coreLoadExecutor-105-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_ c:control_collection   x:collection1] o.a.s.c.c.ZkStateReader path=[/collections/control_collection] [configName]=[conf1] specified config exists in ZooKeeper
   [junit4]   2> 55612 INFO  (coreLoadExecutor-105-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_ c:control_collection   x:collection1] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_CAAE2B1DF4390888-001/control-001/cores/collection1'
   [junit4]   2> 55612 INFO  (coreLoadExecutor-105-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_ c:control_collection   x:collection1] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 55612 INFO  (coreLoadExecutor-105-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_ c:control_collection   x:collection1] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2

[...truncated too long message...]

est(LBHttpSolrClient.java:394)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:730)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]    > 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    > 	... 1 more
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: Server refused connection at: http://127.0.0.1:42892/_/kx/forceleader_test_collection_shard1_replica2
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:615)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:415)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:383)
   [junit4]    > 	... 6 more
   [junit4]    > Caused by: org.apache.http.conn.HttpHostConnectException: Connect to 127.0.0.1:42892 [/127.0.0.1] failed: Connection refused
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:151)
   [junit4]    > 	at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
   [junit4]    > 	at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
   [junit4]    > 	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
   [junit4]    > 	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
   [junit4]    > 	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
   [junit4]    > 	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
   [junit4]    > 	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
   [junit4]    > 	... 10 more
   [junit4]    > Caused by: java.net.ConnectException: Connection refused
   [junit4]    > 	at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]    > 	at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]    > 	at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]    > 	at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]    > 	at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]    > 	at java.net.Socket.connect(Socket.java:589)
   [junit4]    > 	at org.apache.http.conn.socket.PlainConnectionSocketFactory.connectSocket(PlainConnectionSocketFactory.java:74)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:134)
   [junit4]    > 	... 20 more
   [junit4]   2> 3357233 INFO  (SUITE-ForceLeaderTest-seed#[CAAE2B1DF4390888]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_CAAE2B1DF4390888-001
   [junit4]   2> Aug 31, 2016 7:13:15 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {range_facet_l_dv=Lucene50(blocksize=128), _version_=PostingsFormat(name=Memory doPackFST= false), multiDefault=PostingsFormat(name=Direct), a_t=PostingsFormat(name=Direct), intDefault=PostingsFormat(name=Memory doPackFST= false), id=Lucene50(blocksize=128), range_facet_i_dv=PostingsFormat(name=Direct), text=PostingsFormat(name=LuceneVarGapFixedInterval), range_facet_l=PostingsFormat(name=Direct), timestamp=PostingsFormat(name=Direct)}, docValues:{range_facet_l_dv=DocValuesFormat(name=Direct), range_facet_i_dv=DocValuesFormat(name=Lucene54), timestamp=DocValuesFormat(name=Lucene54)}, maxPointsInLeafNode=1438, maxMBSortInHeap=7.336041322335511, sim=ClassicSimilarity, locale=nl-BE, timezone=Africa/Bamako
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=312237992,total=513277952
   [junit4]   2> NOTE: All tests run in this JVM: [TestFoldingMultitermQuery, TestSubQueryTransformerDistrib, BlockJoinFacetRandomTest, CleanupOldIndexTest, CollectionsAPISolrJTest, StandardRequestHandlerTest, TestAnalyzedSuggestions, FileBasedSpellCheckerTest, BasicDistributedZkTest, TestDistribDocBasedVersion, TestSQLHandler, UUIDUpdateProcessorFallbackTest, XsltUpdateRequestHandlerTest, TestRawTransformer, AsyncCallRequestStatusResponseTest, DateRangeFieldTest, DocumentAnalysisRequestHandlerTest, TestDocTermOrds, CacheHeaderTest, DateFieldTest, TestSolrXml, HdfsWriteToMultipleCollectionsTest, TestLegacyNumericRangeQueryBuilder, TestReplicationHandlerBackup, BigEndianAscendingWordDeserializerTest, SolrCloudExampleTest, TestDocSet, JSONWriterTest, TestHighlightDedupGrouping, ConnectionReuseTest, TestTrieFacet, TestDynamicFieldResource, TestPhraseSuggestions, TestSolr4Spatial, TestNumericTerms32, TestManagedSchema, EchoParamsTest, DataDrivenBlockJoinTest, OutputWriterTest, TestRemoteStreaming, QueryElevationComponentTest, CursorPagingTest, WrapperMergePolicyFactoryTest, TestRandomCollapseQParserPlugin, ZkStateReaderTest, ReturnFieldsTest, DeleteInactiveReplicaTest, TestPKIAuthenticationPlugin, TestRecoveryHdfs, TestCopyFieldCollectionResource, IndexSchemaRuntimeFieldTest, TestTolerantUpdateProcessorCloud, SolrRequestParserTest, TestFieldCacheSanityChecker, TestBadConfig, HttpPartitionTest, TestValueSourceCache, TestComponentsName, TestStressLucene, ZkNodePropsTest, QueryResultKeyTest, ResponseHeaderTest, TestUseDocValuesAsStored, OutOfBoxZkACLAndCredentialsProvidersTest, AtomicUpdatesTest, ExplicitHLLTest, DefaultValueUpdateProcessorTest, TestJsonFacets, TriLevelCompositeIdRoutingTest, CloudMLTQParserTest, DocExpirationUpdateProcessorFactoryTest, CollectionStateFormat2Test, CopyFieldTest, AnalysisAfterCoreReloadTest, TestSolrCloudWithSecureImpersonation, DistributedDebugComponentTest, HdfsThreadLeakTest, TestRealTimeGet, TestFaceting, TestDFISimilarityFactory, AddBlockUpdateTest, TestOnReconnectListenerSupport, TestMergePolicyConfig, DistributedFacetPivotLongTailTest, SynonymTokenizerTest, PolyFieldTest, SuggestComponentContextFilterQueryTest, TestRandomFaceting, RAMDirectoryFactoryTest, JavabinLoaderTest, HdfsRecoveryZkTest, SaslZkACLProviderTest, QueryEqualityTest, ExternalFileFieldSortTest, TestHdfsCloudBackupRestore, ResourceLoaderTest, TestChildDocTransformer, PathHierarchyTokenizerFactoryTest, TestDynamicLoading, TestNoOpRegenerator, PreAnalyzedUpdateProcessorTest, TestSchemaResource, SyncSliceTest, TestLazyCores, ChaosMonkeySafeLeaderTest, UnloadDistributedZkTest, OverseerTest, LeaderElectionIntegrationTest, ShardRoutingTest, BasicZkTest, TestReplicationHandler, TestDistributedGrouping, TermVectorComponentDistributedTest, TestJoin, TestStressVersions, TestCoreContainer, SolrCoreTest, StatsComponentTest, TestFiltering, SimplePostToolTest, TestCoreDiscovery, TestExtendedDismaxParser, DocValuesTest, DisMaxRequestHandlerTest, DirectSolrSpellCheckerTest, TestReversedWildcardFilterFactory, TestQueryTypes, TermsComponentTest, RequestHandlersTest, TermVectorComponentTest, TestArbitraryIndexDir, LoggingHandlerTest, TestJmxIntegration, BinaryUpdateRequestHandlerTest, TestPHPSerializedResponseWriter, SearchHandlerTest, UpdateParamsTest, TestSolrIndexConfig, BadComponentTest, TestStressRecovery, SolrTestCaseJ4Test, TestCursorMarkWithoutUniqueKey, TestSimpleTrackingShardHandler, BaseCdcrDistributedZkTest, CdcrBootstrapTest, CloudExitableDirectoryReaderTest, CollectionsAPIAsyncDistributedZkTest, ConcurrentDeleteAndCreateCollectionTest, DeleteShardTest, DistribCursorPagingTest, ForceLeaderTest]
   [junit4] Completed [378/632 (6!)] on J0 in 115.74s, 3 tests, 1 error, 1 skipped <<< FAILURES!

[...truncated 828 lines...]
   [junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20160831_181712_490.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid13708.hprof ...
   [junit4] Heap dump file created [533401820 bytes in 2.740 secs]
   [junit4] <<< JVM J0: EOF ----

[...truncated 11041 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:715: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid13708.hprof

Total time: 342 minutes 11 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1128 - Failure

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1128/

8 tests failed.
FAILED:  org.apache.lucene.index.TestDemoParallelLeafReader.testRandomMultipleSchemaGensSameField

Error Message:
cannot change field "number" from points dimensionCount=1, numBytes=8 to inconsistent dimensionCount=0, numBytes=0

Stack Trace:
java.lang.IllegalArgumentException: cannot change field "number" from points dimensionCount=1, numBytes=8 to inconsistent dimensionCount=0, numBytes=0
	at __randomizedtesting.SeedInfo.seed([7856CAB64594B856:2807C177B262E94F]:0)
	at org.apache.lucene.index.FieldInfo.update(FieldInfo.java:149)
	at org.apache.lucene.index.FieldInfos$Builder.addOrUpdateInternal(FieldInfos.java:412)
	at org.apache.lucene.index.FieldInfos$Builder.add(FieldInfos.java:432)
	at org.apache.lucene.index.SegmentMerger.mergeFieldInfos(SegmentMerger.java:186)
	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:84)
	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4287)
	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3864)
	at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
	at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
	at org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3047)
	at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3181)
	at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3146)
	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.commit(TestDemoParallelLeafReader.java:160)
	at org.apache.lucene.index.TestDemoParallelLeafReader.testRandomMultipleSchemaGensSameField(TestDemoParallelLeafReader.java:1091)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationStartStop

Error Message:
Timeout while trying to assert number of documents @ target_collection

Stack Trace:
java.lang.AssertionError: Timeout while trying to assert number of documents @ target_collection
	at __randomizedtesting.SeedInfo.seed([96E63FAC05C47E13:1525E13F111CF59A]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:273)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationStartStop(CdcrReplicationDistributedZkTest.java:173)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.AssertionError: expected:<10> but was:<4>
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:264)
	... 42 more


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testResilienceWithDeleteByQueryOnTarget

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard1

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard1
	at __randomizedtesting.SeedInfo.seed([96E63FAC05C47E13:3684AD36D59DBB56]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testResilienceWithDeleteByQueryOnTarget(CdcrReplicationDistributedZkTest.java:588)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.core.HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig

Error Message:
The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~134.217728 MB.

Stack Trace:
java.lang.RuntimeException: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~134.217728 MB.
	at __randomizedtesting.SeedInfo.seed([96E63FAC05C47E13:6149F687F84D9438]:0)
	at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:304)
	at org.apache.solr.core.HdfsDirectoryFactory.getBlockDirectoryCache(HdfsDirectoryFactory.java:280)
	at org.apache.solr.core.HdfsDirectoryFactory.create(HdfsDirectoryFactory.java:220)
	at org.apache.solr.core.HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig(HdfsDirectoryFactoryTest.java:108)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.OutOfMemoryError: Direct buffer memory
	at java.nio.Bits.reserveMemory(Bits.java:693)
	at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123)
	at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)
	at org.apache.solr.store.blockcache.BlockCache.<init>(BlockCache.java:63)
	at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:302)
	... 42 more


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestStressReplication

Error Message:
timed out waiting for collection1 startAt time to exceed: Thu Oct 13 11:49:29 OMST 2016

Stack Trace:
java.lang.AssertionError: timed out waiting for collection1 startAt time to exceed: Thu Oct 13 11:49:29 OMST 2016
	at __randomizedtesting.SeedInfo.seed([96E63FAC05C47E13:4D4D3F6A00EC17A0]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.handler.TestReplicationHandler.watchCoreStartAt(TestReplicationHandler.java:1508)
	at org.apache.solr.handler.TestReplicationHandler.doTestStressReplication(TestReplicationHandler.java:858)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:830)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:830)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.

[...truncated too long message...]

.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-collective-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-event-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/bcprov-jdk15-1.45.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/cglib-nodep-2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/commons-math3-3.4.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/easymock-3.0.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-common-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-hdfs-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-minikdc-2.7.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-annotations-2.5.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-databind-2.5.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/mockito-core-1.9.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/objenesis-1.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/presto-parser-0.122.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/slice-0.10.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/analysis/icu/lucene-analyzers-icu-7.0.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/analysis-extras/lib/icu4j-56.1.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_102/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.3.4.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20161013_051024_079.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20161013_051024_079.suites -stdin
   [junit4] ERROR: JVM J1 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1501)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:121)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:955)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:952)
   [junit4] 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4] 	at java.lang.Thread.run(Thread.java:745)

BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:773: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:709: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build.xml:252: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/common-build.xml:536: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:1443: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:984: At least one slave process threw an exception, first: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.

Total time: 255 minutes 38 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1127 - Still unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1127/

8 tests failed.
FAILED:  org.apache.lucene.search.TestFuzzyQuery.testRandom

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([DC469116CE6EC137]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.search.TestFuzzyQuery

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([DC469116CE6EC137]:0)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchAddsWithDelete

Error Message:
Timeout while trying to assert number of documents @ target_collection

Stack Trace:
java.lang.AssertionError: Timeout while trying to assert number of documents @ target_collection
	at __randomizedtesting.SeedInfo.seed([1CF269DB07F9BC4A:6BE8107FAFE0BD66]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:273)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchAddsWithDelete(CdcrReplicationDistributedZkTest.java:532)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.AssertionError: expected:<59> but was:<33>
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:264)
	... 42 more


FAILED:  org.apache.solr.cloud.hdfs.StressHdfsTest.test

Error Message:
Could not find collection:delete_data_dir

Stack Trace:
java.lang.AssertionError: Could not find collection:delete_data_dir
	at __randomizedtesting.SeedInfo.seed([1CF269DB07F9BC4A:94A65601A905D1B2]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:153)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:138)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:133)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:856)
	at org.apache.solr.cloud.hdfs.StressHdfsTest.createAndDeleteCollection(StressHdfsTest.java:159)
	at org.apache.solr.cloud.hdfs.StressHdfsTest.test(StressHdfsTest.java:103)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:830)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:830)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:830)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:830)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:830)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carr

[...truncated too long message...]

nterceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:430)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
   [junit4]   2> 	at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
   [junit4]   2> 	at org.apache.hadoop.metrics2.util.MBeans.unregister(MBeans.java:81)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stopMBeans(MetricsSourceAdapter.java:226)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stop(MetricsSourceAdapter.java:211)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stopSources(MetricsSystemImpl.java:463)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
   [junit4]   2> 	at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
   [junit4]   2> 	at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
   [junit4]   2> 	at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:198)
   [junit4]   2> 	at org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.teardownClass(HdfsCollectionsAPIDistributedZkTest.java:52)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: java.lang.NullPointerException
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
   [junit4]   2> 	... 54 more
   [junit4]   2> 4893991 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1CF269DB07F9BC4A]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_1CF269DB07F9BC4A-001
   [junit4]   2> Oct 12, 2016 8:49:32 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 35 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=6185, maxDocsPerChunk=8, blockSize=992), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=6185, blockSize=992)), sim=RandomSimilarity(queryNorm=false): {}, locale=es-NI, timezone=Pacific/Wake
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=2,free=153867744,total=482869248
   [junit4]   2> NOTE: All tests run in this JVM: [HdfsNNFailoverTest, BlockDirectoryTest, TestDistribIDF, DistributedFacetPivotLongTailTest, TestQuerySenderNoQuery, MultiThreadedOCPTest, TestSolrConfigHandlerConcurrent, PrimUtilsTest, WrapperMergePolicyFactoryTest, HdfsChaosMonkeyNothingIsSafeTest, TestLRUStatsCache, TestPhraseSuggestions, SolrPluginUtilsTest, TestLeaderElectionZkExpiry, HdfsTlogReplayBufferedWhileIndexingTest, XmlUpdateRequestHandlerTest, TestReplicaProperties, GraphQueryTest, ParsingFieldUpdateProcessorsTest, TestGraphMLResponseWriter, DistributedExpandComponentTest, TestCloudSchemaless, HLLSerializationTest, EchoParamsTest, CollectionStateFormat2Test, TestBulkSchemaConcurrent, RAMDirectoryFactoryTest, DistribCursorPagingTest, TestReversedWildcardFilterFactory, TestSolrJ, CoreMergeIndexesAdminHandlerTest, TolerantUpdateProcessorTest, TestTolerantSearch, PreAnalyzedFieldTest, TestRandomCollapseQParserPlugin, SuggesterWFSTTest, SliceStateTest, TestReplicationHandlerBackup, CdcrUpdateLogTest, TestSchemaSimilarityResource, FullSolrCloudDistribCmdsTest, TestConfigSetsAPI, TestJsonFacetRefinement, DirectUpdateHandlerOptimizeTest, TestTolerantUpdateProcessorRandomCloud, TestGraphTermsQParserPlugin, TestSolrDeletionPolicy1, TestExtendedDismaxParser, TestIBSimilarityFactory, AssignTest, TestIndexingPerformance, TestAuthenticationFramework, DistributedFacetPivotWhiteBoxTest, TestManagedStopFilterFactory, ConnectionManagerTest, SegmentsInfoRequestHandlerTest, TestSolrXml, JavabinLoaderTest, BooleanFieldTest, HdfsLockFactoryTest, DocValuesMissingTest, TestWriterPerf, DistributedQueueTest, TestSolrFieldCacheMBean, CdcrBootstrapTest, TestCSVResponseWriter, TestSolrCloudWithDelegationTokens, TestFreeTextSuggestions, TestUtils, TestRandomFlRTGCloud, ShowFileRequestHandlerTest, LeaderInitiatedRecoveryOnCommitTest, TestComponentsName, TestAuthorizationFramework, TestSolrQueryParser, ConnectionReuseTest, HighlighterConfigTest, TestFieldTypeResource, StatsComponentTest, SuggesterTSTTest, RecoveryZkTest, TestGroupingSearch, TestManagedResource, SimpleFacetsTest, TestInitParams, OverseerTest, AddSchemaFieldsUpdateProcessorFactoryTest, PrimitiveFieldTypeTest, TestConfigSetProperties, TestLMDirichletSimilarityFactory, MBeansHandlerTest, TestSimpleQParserPlugin, TestExpandComponent, CleanupOldIndexTest, CurrencyFieldXmlFileTest, OverseerCollectionConfigSetProcessorTest, TestLegacyFieldCache, AnalyticsQueryTest, TestLegacyNumericRangeQueryBuilder, TestDefaultSearchFieldResource, TestBulkSchemaAPI, TestFastOutputStream, UpdateParamsTest, BasicDistributedZkTest, BasicDistributedZk2Test, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest, TestCoreDiscovery, CoreAdminHandlerTest, SuggesterTest, SpatialFilterTest, PolyFieldTest, WordBreakSolrSpellCheckerTest, FieldMutatingUpdateProcessorTest, DefaultValueUpdateProcessorTest, DocValuesMultiTest, DistanceFunctionTest, SolrInfoMBeanTest, DebugComponentTest, LukeRequestHandlerTest, StandardRequestHandlerTest, FileBasedSpellCheckerTest, TestValueSourceCache, PathHierarchyTokenizerFactoryTest, MoreLikeThisHandlerTest, TestJmxIntegration, ReturnFieldsTest, TestAnalyzedSuggestions, TestPartialUpdateDeduplication, PingRequestHandlerTest, TestLFUCache, SolrTestCaseJ4Test, TestCrossCoreJoin, TestCursorMarkWithoutUniqueKey, TestDistributedMissingSort, CdcrRequestHandlerTest, CloudExitableDirectoryReaderTest, CollectionReloadTest, CollectionsAPISolrJTest, DeleteReplicaTest, LeaderInitiatedRecoveryOnShardRestartTest, OverseerRolesTest, RollingRestartTest, TestDistribDocBasedVersion, TestExclusionRuleCollectionAccess, TestLeaderRecoverFromLogOnStartup, TestSSLRandomization, TestSizeLimitedDistributedMap, TestSolrCloudWithSecureImpersonation, TestStressLiveNodes, VMParamsZkACLAndCredentialsProvidersTest, HdfsBasicDistributedZk2Test, HdfsChaosMonkeySafeLeaderTest, HdfsCollectionsAPIDistributedZkTest]
   [junit4] Completed [571/638 (5!)] on J2 in 949.50s, 1 test, 1 error <<< FAILURES!

[...truncated 60518 lines...]



[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1126 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1126/

No tests ran.

Build Log:
[...truncated 11625 lines...]
   [junit4] JVM J1: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20161005_083740_205.sysout
   [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid6760.hprof ...
   [junit4] Heap dump file created [714278651 bytes in 16.125 secs]
   [junit4] <<< JVM J1: EOF ----

   [junit4] JVM J1: stderr was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20161005_083740_205.syserr
   [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
   [junit4] WARN: Unhandled exception in event serialization. -> java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] <<< JVM J1: EOF ----

[...truncated 32 lines...]
   [junit4] Suite: org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/init-core-data-001
   [junit4]   2> 1786063 INFO  (SUITE-ConcurrentDeleteAndCreateCollectionTest-seed#[BF6C5C74A9CEFC3D]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 1786071 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[BF6C5C74A9CEFC3D]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testConcurrentCreateAndDeleteOverTheSameConfig
   [junit4]   2> 1786071 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[BF6C5C74A9CEFC3D]) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001
   [junit4]   2> 1786071 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[BF6C5C74A9CEFC3D]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1786076 INFO  (Thread-1725) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1786076 INFO  (Thread-1725) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1786179 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[BF6C5C74A9CEFC3D]) [    ] o.a.s.c.ZkTestServer start zk server on port:58815
   [junit4]   2> 1786239 INFO  (jetty-launcher-799-thread-1) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 1786283 INFO  (jetty-launcher-799-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@d571903{/solr,null,AVAILABLE}
   [junit4]   2> 1786286 INFO  (jetty-launcher-799-thread-1) [    ] o.e.j.s.ServerConnector Started ServerConnector@3d714097{SSL,[ssl, http/1.1]}{127.0.0.1:56041}
   [junit4]   2> 1786286 INFO  (jetty-launcher-799-thread-1) [    ] o.e.j.s.Server Started @1797249ms
   [junit4]   2> 1786286 INFO  (jetty-launcher-799-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=56041}
   [junit4]   2> 1786286 INFO  (jetty-launcher-799-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.0.0
   [junit4]   2> 1786287 INFO  (jetty-launcher-799-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1786287 INFO  (jetty-launcher-799-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1786287 INFO  (jetty-launcher-799-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2016-10-05T09:07:37.802Z
   [junit4]   2> 1786344 INFO  (jetty-launcher-799-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1786404 INFO  (jetty-launcher-799-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:58815/solr
   [junit4]   2> 1786528 INFO  (jetty-launcher-799-thread-1) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:56041_solr
   [junit4]   2> 1786529 INFO  (jetty-launcher-799-thread-1) [n:127.0.0.1:56041_solr    ] o.a.s.c.Overseer Overseer (id=96708752676749315-127.0.0.1:56041_solr-n_0000000000) starting
   [junit4]   2> 1786580 INFO  (jetty-launcher-799-thread-1) [n:127.0.0.1:56041_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56041_solr
   [junit4]   2> 1786591 INFO  (zkCallback-805-thread-1-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1786607 INFO  (jetty-launcher-799-thread-1) [n:127.0.0.1:56041_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/.
   [junit4]   2> 1786937 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=testconfig&name=collection0&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1786948 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=testconfig&name=collection1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1786983 INFO  (OverseerThreadFactory-2254-thread-1-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection0
   [junit4]   2> 1786992 INFO  (OverseerThreadFactory-2254-thread-2-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 1787172 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=testconfig&name=collection0_shard1_replica1&action=CREATE&numShards=1&collection=collection0&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1787254 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=testconfig&name=collection1_shard1_replica1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1787280 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1787395 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1788222 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1788239 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.s.IndexSchema [collection0_shard1_replica1] Schema name=minimal
   [junit4]   2> 1788246 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1788247 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'collection0_shard1_replica1' using configuration from collection collection0
   [junit4]   2> 1788252 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore [[collection0_shard1_replica1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/collection0_shard1_replica1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/./collection0_shard1_replica1/data/]
   [junit4]   2> 1788282 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1788304 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.IndexSchema [collection1_shard1_replica1] Schema name=minimal
   [junit4]   2> 1788306 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1788317 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica1' using configuration from collection collection1
   [junit4]   2> 1788318 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [[collection1_shard1_replica1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/collection1_shard1_replica1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/./collection1_shard1_replica1/data/]
   [junit4]   2> 1788335 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1788336 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1788337 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@391d9575[collection0_shard1_replica1] main]
   [junit4]   2> 1788355 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/testconfig
   [junit4]   2> 1788359 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/testconfig
   [junit4]   2> 1788360 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1788360 INFO  (searcherExecutor-2257-thread-1-processing-n:127.0.0.1:56041_solr x:collection0_shard1_replica1 s:shard1 c:collection0 r:core_node1) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore [collection0_shard1_replica1] Registered new searcher Searcher@391d9575[collection0_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1788388 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1788388 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1788390 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7fce5056[collection1_shard1_replica1] main]
   [junit4]   2> 1788395 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1788395 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1788395 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:56041/solr/collection0_shard1_replica1/
   [junit4]   2> 1788396 ERROR (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SyncStrategy No UpdateLog found - cannot sync
   [junit4]   2> 1788402 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 1788406 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/testconfig
   [junit4]   2> 1788408 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/testconfig
   [junit4]   2> 1788408 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1788408 INFO  (searcherExecutor-2258-thread-1-processing-n:127.0.0.1:56041_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [collection1_shard1_replica1] Registered new searcher Searcher@7fce5056[collection1_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1788416 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:56041/solr/collection0_shard1_replica1/ shard1
   [junit4]   2> 1788420 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1788420 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1788420 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:56041/solr/collection1_shard1_replica1/
   [junit4]   2> 1788420 ERROR (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SyncStrategy No UpdateLog found - cannot sync
   [junit4]   2> 1788421 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 1788429 INFO  (zkCallback-805-thread-2-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1788433 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:56041/solr/collection1_shard1_replica1/ shard1
   [junit4]   2> 1788471 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1788485 INFO  (zkCallback-805-thread-2-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1788486 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=testconfig&name=collection0_shard1_replica1&action=CREATE&numShards=1&collection=collection0&shard=shard1&wt=javabin&version=2} status=0 QTime=1314
   [junit4]   2> 1788503 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1788535 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1788543 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=testconfig&name=collection1_shard1_replica1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2} status=0 QTime=1289
   [junit4]   2> 1788546 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1788547 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1788685 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1788991 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1788991 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1789503 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=testconfig&name=collection0&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2566
   [junit4]   2> 1789506 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=collection0&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1789510 INFO  (OverseerThreadFactory-2254-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 1789511 INFO  (qtp1908042262-5757) [n:127.0.0.1:56041_solr    ] o.a.s.c.SolrCore [collection0_shard1_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@33e50332
   [junit4]   2> 1789535 INFO  (qtp1908042262-5757) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=collection0_shard1_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=24
   [junit4]   2> 1789547 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=testconfig&name=collection1&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2599
   [junit4]   2> 1789552 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=collection1&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1789571 INFO  (OverseerThreadFactory-2254-thread-4-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 1789580 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr    ] o.a.s.c.SolrCore [collection1_shard1_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@79482cd9
   [junit4]   2> 1789594 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=collection1_shard1_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=14
   [junit4]   2> 1790273 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=collection0&action=DELETE&wt=javabin&version=2} status=0 QTime=767
   [junit4]   2> 1790274 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=testconfig&name=collection0&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1790288 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1790288 INFO  (OverseerThreadFactory-2254-thread-5-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection0
   [junit4]   2> 1790449 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=collection1&action=DELETE&wt=javabin&version=2} status=0 QTime=897
   [junit4]   2> 1790452 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=testconfig&name=collection1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1790484 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1790486 INFO  (OverseerThreadFactory-2254-thread-4-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 1790500 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=testconfig&name=collection0_shard1_replica1&action=CREATE&numShards=1&collection=collection0&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1790590 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=testconfig&name=collection1_shard1_replica1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1790592 INFO  (zkCallback-805-thread-1-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1790696 INFO  (zkCallback-805-thread-1-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1791543 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1791558 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.s.IndexSchema [collection0_shard1_replica1] Schema name=minimal
   [junit4]   2> 1791560 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1791562 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'collection0_shard1_replica1' using configuration from collection collection0
   [junit4]   2> 1791562 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore [[collection0_shard1_replica1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/collection0_shard1_replica1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/./collection0_shard1_replica1/data/]
   [junit4]   2> 1791575 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1791576 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1791580 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@aac94c4[collection0_shard1_replica1] main]
   [junit4]   2> 1791581 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/testconfig
   [junit4]   2> 1791582 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/testconfig
   [junit4]   2> 1791582 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1791582 INFO  (searcherExecutor-2265-thread-1-processing-n:127.0.0.1:56041_solr x:collection0_shard1_replica1 s:shard1 c:collection0 r:core_node1) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore [collection0_shard1_replica1] Registered new searcher Searcher@aac94c4[collection0_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1791587 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1791588 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1791588 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:56041/solr/collection0_shard1_replica1/
   [junit4]   2> 1791588 ERROR (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SyncStrategy No UpdateLog found - cannot sync
   [junit4]   2> 1791588 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 1791600 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1791601 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:56041/solr/collection0_shard1_replica1/ shard1
   [junit4]   2> 1791608 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.IndexSchema [collection1_shard1_replica1] Schema name=minimal
   [junit4]   2> 1791610 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1791611 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica1' using configuration from collection collection1
   [junit4]   2> 1791612 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [[collection1_shard1_replica1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/collection1_shard1_replica1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/./collection1_shard1_replica1/data/]
   [junit4]   2> 1791623 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1791623 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1791624 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@35b94579[collection1_shard1_replica1] main]
   [junit4]   2> 1791625 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/testconfig
   [junit4]   2> 1791625 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/testconfig
   [junit4]   2> 1791625 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1791626 INFO  (searcherExecutor-2269-thread-1-processing-n:127.0.0.1:56041_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [collection1_shard1_replica1] Registered new searcher Searcher@35b94579[collection1_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1791631 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1791632 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1791632 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1791632 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:56041/solr/collection1_shard1_replica1/
   [junit4]   2> 1791632 ERROR (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SyncStrategy No UpdateLog found - cannot sync
   [junit4]   2> 1791632 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 1791638 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:56041/solr/collection1_shard1_replica1/ shard1
   [junit4]   2> 1791652 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1791655 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=testconfig&name=collection0_shard1_replica1&action=CREATE&numShards=1&collection=collection0&shard=shard1&wt=javabin&version=2} status=0 QTime=1155
   [junit4]   2> 1791657 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1791668 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1791691 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1791700 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1791703 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=testconfig&name=collection1_shard1_replica1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2} status=0 QTime=1113
   [junit4]   2> 1791709 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1791804 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1792488 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000008 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1792488 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000010 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1792668 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=testconfig&name=collection0&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2394
   [junit4]   2> 1792669 INFO  (qtp1908042262-5757) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=collection0&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1792680 INFO  (OverseerThreadFactory-2254-thread-4-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 1792682 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr    ] o.a.s.c.SolrCore [collection0_shard1_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@4e26518e
   [junit4]   2> 1792689 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=collection0_shard1_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=8
   [junit4]   2> 1792711 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=testconfig&name=collection1&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2259
   [junit4]   2> 1792712 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=collection1&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1792736 INFO  (OverseerThreadFactory-2254-thread-5-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 1792737 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr    ] o.a.s.c.SolrCore [collection1_shard1_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@f4d8dc4
   [junit4]   2> 1792740 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=collection1_shard1_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=2
   [junit4]   2> 1793301 INFO  (qtp1908042262-5757) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=collection0&action=DELETE&wt=javabin&version=2} status=0 QTime=631
   [junit4]   2> 1793303 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=testconfig&name=collection0&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1793340 INFO  (OverseerThreadFactory-2254-thread-4-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection0
   [junit4]   2> 1793348 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000012 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1793453 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=collection1&action=DELETE&wt=javabin&version=2} status=0 QTime=740
   [junit4]   2> 1793454 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=testconfig&name=collection1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1793463 INFO  (OverseerThreadFactory-2254-thread-5-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 1793464 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000014 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1793572 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=testconfig&name=collection0_shard1_replica1&action=CREATE&numShards=1&collection=collection0&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1793676 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=testconfig&name=collection1_shard1_replica1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1793679 INFO  (zkCallback-805-thread-1-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1793783 INFO  (zkCallback-805-thread-1-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1794585 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1794649 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.s.IndexSchema [collection0_shard1_replica1] Schema name=minimal
   [junit4]   2> 1794660 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1794661 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'collection0_shard1_replica1' using configuration from collection collection0
   [junit4]   2> 1794662 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore [[collection0_shard1_replica1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/collection0_shard1_replica1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/./collection0_shard1_replica1/data/]
   [junit4]   2> 1794692 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1794701 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.IndexSchema [collection1_shard1_replica1] Schema name=minimal
   [junit4]   2> 1794703 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1794706 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica1' using configuration from collection collection1
   [junit4]   2> 1794706 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [[collection1_shard1_replica1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/collection1_shard1_replica1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/./collection1_shard1_replica1/data/]
   [junit4]   2> 1794708 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1794708 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1794711 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5afa98bf[collection0_shard1_replica1] main]
   [junit4]   2> 1794718 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1794718 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1794718 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@577b58e0[collection1_shard1_replica1] main]
   [junit4]   2> 1794720 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/testconfig
   [junit4]   2> 1794721 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/testconfig
   [junit4]   2> 1794721 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1794732 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1794732 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1794732 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:56041/solr/collection1_shard1_replica1/
   [junit4]   2> 1794732 ERROR (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SyncStrategy No UpdateLog found - cannot sync
   [junit4]   2> 1794732 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 1794736 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:56041/solr/collection1_shard1_replica1/ shard1
   [junit4]   2> 1794737 INFO  (searcherExecutor-2275-thread-1-processing-n:127.0.0.1:56041_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [collection1_shard1_replica1] Registered new searcher Searcher@577b58e0[collection1_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1794738 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/testconfig
   [junit4]   2> 1794738 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/testconfig
   [junit4]   2> 1794739 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1794739 INFO  (searcherExecutor-2273-thread-1-processing-n:127.0.0.1:56041_solr x:collection0_shard1_replica1 s:shard1 c:collection0 r:core_node1) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore [collection0_shard1_replica1] Registered new searcher Searcher@5afa98bf[collection0_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1794744 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1794748 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1794748 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1794748 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:56041/solr/collection0_shard1_replica1/
   [junit4]   2> 1794748 ERROR (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SyncStrategy No UpdateLog found - cannot sync
   [junit4]   2> 1794748 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 1794759 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:56041/solr/collection0_shard1_replica1/ shard1
   [junit4]   2> 1794787 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1794789 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=testconfig&name=collection1_shard1_replica1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2} status=0 QTime=1113
   [junit4]   2> 1794790 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1794797 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1794815 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1794821 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=testconfig&name=collection0_shard1_replica1&action=CREATE&numShards=1&collection=collection0&shard=shard1&wt=javabin&version=2} status=0 QTime=1249
   [junit4]   2> 1794822 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1794829 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1794926 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1795468 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000016 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1795468 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000018 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1795798 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=testconfig&name=collection1&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2343
   [junit4]   2> 1795799 INFO  (qtp1908042262-5757) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=collection1&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1795822 INFO  (OverseerThreadFactory-2254-thread-4-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 1795823 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr    ] o.a.s.c.SolrCore [collection1_shard1_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@7f736999
   [junit4]   2> 1795826 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=collection1_shard1_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=3
   [junit4]   2> 1795829 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=testconfig&name=collection0&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2526
   [junit4]   2> 1795830 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=collection0&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1795852 INFO  (OverseerThreadFactory-2254-thread-5-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 1795853 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr    ] o.a.s.c.SolrCore [collection0_shard1_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@639add30
   [junit4]   2> 1795860 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=collection0_shard1_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=7
   [junit4]   2> 1796444 INFO  (qtp1908042262-5757) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=collection1&action=DELETE&wt=javabin&version=2} status=0 QTime=645
   [junit4]   2> 1796445 INFO  (qtp1908042262-5757) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=testconfig&name=collection1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1796452 INFO  (OverseerThreadFactory-2254-thread-4-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 1796453 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000020 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1796573 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=collection0&action=DELETE&wt=javabin&version=2} status=0 QTime=742
   [junit4]   2> 1796574 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=testconfig&name=collection0&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1796576 INFO  (OverseerThreadFactory-2254-thread-5-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection0
   [junit4]   2> 1796577 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000022 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1796691 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=testconfig&name=collection1_shard1_replica1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1796893 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=testconfig&name=collection0_shard1_replica1&action=CREATE&numShards=1&collection=collection0&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1796896 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1797000 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1797753 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1797774 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.IndexSchema [collection1_shard1_replica1] Schema name=minimal
   [junit4]   2> 1797785 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1797786 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica1' using configuration from collection collection1
   [junit4]   2> 1797787 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [[collection1_shard1_replica1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/collection1_shard1_replica1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/./collection1_shard1_replica1/data/]
   [junit4]   2> 1797817 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1797817 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1797827 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@47882b23[collection1_shard1_replica1] main]
   [junit4]   2> 1797828 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/testconfig
   [junit4]   2> 1797828 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/testconfig
   [junit4]   2> 1797828 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1797835 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1797836 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1797836 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:56041/solr/collection1_shard1_replica1/
   [junit4]   2> 1797836 ERROR (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SyncStrategy No UpdateLog found - cannot sync
   [junit4]   2> 1797836 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 1797846 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:56041/solr/collection1_shard1_replica1/ shard1
   [junit4]   2> 1797848 INFO  (searcherExecutor-2281-thread-1-processing-n:127.0.0.1:56041_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [collection1_shard1_replica1] Registered new searcher Searcher@47882b23[collection1_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1797903 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1797915 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.s.IndexSchema [collection0_shard1_replica1] Schema name=minimal
   [junit4]   2> 1797921 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1797922 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'collection0_shard1_replica1' using configuration from collection collection0
   [junit4]   2> 1797923 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore [[collection0_shard1_replica1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/collection0_shard1_replica1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/./collection0_shard1_replica1/data/]
   [junit4]   2> 1797935 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1797936 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1797936 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6f39c0ba[collection0_shard1_replica1] main]
   [junit4]   2> 1797937 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/testconfig
   [junit4]   2> 1797938 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/testconfig
   [junit4]   2> 1797938 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1797943 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1797943 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1797943 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:56041/solr/collection0_shard1_replica1/
   [junit4]   2> 1797943 ERROR (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SyncStrategy No UpdateLog found - cannot sync
   [junit4]   2> 1797944 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 1797944 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1797945 INFO  (searcherExecutor-2285-thread-1-processing-n:127.0.0.1:56041_solr x:collection0_shard1_replica1 s:shard1 c:collection0 r:core_node1) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore [collection0_shard1_replica1] Registered new searcher Searcher@6f39c0ba[collection0_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1797949 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1797951 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:56041/solr/collection0_shard1_replica1/ shard1
   [junit4]   2> 1797961 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=testconfig&name=collection1_shard1_replica1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2} status=0 QTime=1270
   [junit4]   2> 1797964 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1797965 INFO  (qtp1908042262-5757) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1798007 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1798013 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1798024 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=testconfig&name=collection0_shard1_replica1&action=CREATE&numShards=1&collection=collection0&shard=shard1&wt=javabin&version=2} status=0 QTime=1131
   [junit4]   2> 1798027 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1798117 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1798581 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000024 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1798581 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000026 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1798966 INFO  (qtp1908042262-5757) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=testconfig&name=collection1&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2520
   [junit4]   2> 1798967 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=collection1&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1798969 INFO  (OverseerThreadFactory-2254-thread-5-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 1798970 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr    ] o.a.s.c.SolrCore [collection1_shard1_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@3f6c5682
   [junit4]   2> 1798975 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=collection1_shard1_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=5
   [junit4]   2> 1799027 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=testconfig&name=collection0&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2453
   [junit4]   2> 1799029 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=collection0&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1799031 INFO  (OverseerThreadFactory-2254-thread-4-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 1799032 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr    ] o.a.s.c.SolrCore [collection0_shard1_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@5aae268b
   [junit4]   2> 1799040 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=collection0_shard1_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=7
   [junit4]   2> 1799586 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=collection1&action=DELETE&wt=javabin&version=2} status=0 QTime=619
   [junit4]   2> 1799592 INFO  (qtp1908042262-5757) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=testconfig&name=collection1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1799594 INFO  (OverseerThreadFactory-2254-thread-5-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 1799595 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000028 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1799770 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=collection0&action=DELETE&wt=javabin&version=2} status=0 QTime=741
   [junit4]   2> 1799772 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=testconfig&name=collection0&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1799784 INFO  (OverseerThreadFactory-2254-thread-4-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection0
   [junit4]   2> 1799785 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000030 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1799822 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=testconfig&name=collection1_shard1_replica1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1799889 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=testconfig&name=collection0_shard1_replica1&action=CREATE&numShards=1&collection=collection0&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1799893 INFO  (zkCallback-805-thread-1-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1799997 INFO  (zkCallback-805-thread-1-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1800833 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1800841 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.IndexSchema [collection1_shard1_replica1] Schema name=minimal
   [junit4]   2> 1800852 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1800854 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica1' using configuration from collection collection1
   [junit4]   2> 1800854 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [[collection1_shard1_replica1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/collection1_shard1_replica1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/./collection1_shard1_replica1/data/]
   [junit4]   2> 1800872 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1800872 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1800874 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3f73c646[collection1_shard1_replica1] main]
   [junit4]   2> 1800875 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/testconfig
   [junit4]   2> 1800876 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/testconfig
   [junit4]   2> 1800876 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1800886 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1800886 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1800886 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:56041/solr/collection1_shard1_replica1/
   [junit4]   2> 1800886 ERROR (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SyncStrategy No UpdateLog found - cannot sync
   [junit4]   2> 1800886 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 1800886 INFO  (searcherExecutor-2289-thread-1-processing-n:127.0.0.1:56041_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [collection1_shard1_replica1] Registered new searcher Searcher@3f73c646[collection1_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1800908 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1800962 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:56041/solr/collection1_shard1_replica1/ shard1
   [junit4]   2> 1800973 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.s.IndexSchema [collection0_shard1_replica1] Schema name=minimal
   [junit4]   2> 1800975 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1800976 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'collection0_shard1_replica1' using configuration from collection collection0
   [junit4]   2> 1800976 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore [[collection0_shard1_replica1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/collection0_shard1_replica1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/./collection0_shard1_replica1/data/]
   [junit4]   2> 1800987 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1800988 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1800988 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@351b4b32[collection0_shard1_replica1] main]
   [junit4]   2> 1800989 I

[...truncated too long message...]

ntAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: java.lang.NullPointerException
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
   [junit4]   2> 	... 54 more
   [junit4]   2> 15655648 INFO  (SUITE-StressHdfsTest-seed#[BF6C5C74A9CEFC3D]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.StressHdfsTest_BF6C5C74A9CEFC3D-001
   [junit4]   2> Oct 05, 2016 12:58:47 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene70, sim=RandomSimilarity(queryNorm=false): {}, locale=zh-SG, timezone=Pacific/Honolulu
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=2,free=294931632,total=506462208
   [junit4]   2> NOTE: All tests run in this JVM: [ExplicitHLLTest, CoreSorterTest, PluginInfoTest, TestLegacyNumericRangeQueryBuilder, AtomicUpdatesTest, DistributedFacetPivotLongTailTest, TestGraphTermsQParserPlugin, BufferStoreTest, QueryElevationComponentTest, JavabinLoaderTest, TlogReplayBufferedWhileIndexingTest, RequestLoggingTest, QueryEqualityTest, TestRealTimeGet, CollectionTooManyReplicasTest, TestFieldTypeResource, TestManagedSchemaAPI, TestHighFrequencyDictionaryFactory, TestLockTree, TestClusterStateMutator, TestRangeQuery, BasicAuthIntegrationTest, SearchHandlerTest, RegexBoostProcessorTest, DeleteShardTest, TestStressLucene, TestConfigOverlay, BigEndianAscendingWordDeserializerTest, BasicDistributedZkTest, TestSolrCoreProperties, HdfsDirectoryFactoryTest, TestDFISimilarityFactory, TestHdfsCloudBackupRestore, SolrCoreTest, TestSortByMinMaxFunction, TestSubQueryTransformer, TestCustomSort, TestLeaderInitiatedRecoveryThread, SuggestComponentContextFilterQueryTest, TestRandomRequestDistribution, NumericFieldsTest, TestValueSourceCache, ResponseLogComponentTest, TestSchemaSimilarityResource, UniqFieldsUpdateProcessorFactoryTest, TestSolrFieldCacheMBean, TestBadConfig, TestFieldCollectionResource, TestExpandComponent, HdfsRecoverLeaseTest, TestShortCircuitedRequests, TestRequestForwarding, TestOmitPositions, TestSolrDeletionPolicy2, TestRTimerTree, SystemInfoHandlerTest, ReturnFieldsTest, SyncSliceTest, TestGraphMLResponseWriter, SolrCloudExampleTest, DisMaxRequestHandlerTest, TestAtomicUpdateErrorCases, DistributedSpellCheckComponentTest, TestLFUCache, BlockJoinFacetRandomTest, ConcurrentDeleteAndCreateCollectionTest, TestDocTermOrds, TestConfigSetsAPIExclusivity, RestartWhileUpdatingTest, TestCSVLoader, TestDistribDocBasedVersion, CreateCollectionCleanupTest, CursorPagingTest, ResponseHeaderTest, TestLRUCache, TestStressLiveNodes, NotRequiredUniqueKeyTest, DateMathParserTest, ClusterStateTest, BooleanFieldTest, RankQueryTest, PolyFieldTest, DocValuesMissingTest, ZkNodePropsTest, TestNumericTerms32, TestFieldCacheVsDocValues, TestLMJelinekMercerSimilarityFactory, FastVectorHighlighterTest, CoreMergeIndexesAdminHandlerTest, ScriptEngineTest, AnalyticsQueryTest, TestRemoteStreaming, AsyncCallRequestStatusResponseTest, TestHdfsBackupRestoreCore, DistributedFacetPivotWhiteBoxTest, TestDistributedSearch, TestManagedResource, TestSolrQueryResponse, TestManagedStopFilterFactory, TestMiniSolrCloudClusterSSL, SpellCheckCollatorTest, TestFoldingMultitermQuery, SuggesterWFSTTest, SolrCoreCheckLockOnStartupTest, TestPseudoReturnFields, DefaultValueUpdateProcessorTest, SortByFunctionTest, DocValuesMultiTest, DistanceFunctionTest, SolrInfoMBeanTest, XsltUpdateRequestHandlerTest, LukeRequestHandlerTest, TestQueryTypes, FileBasedSpellCheckerTest, PathHierarchyTokenizerFactoryTest, MoreLikeThisHandlerTest, IndexSchemaRuntimeFieldTest, IndexSchemaTest, TestCollationField, QueryParsingTest, TestAnalyzedSuggestions, TestPHPSerializedResponseWriter, CSVRequestHandlerTest, SOLR749Test, TestQuerySenderListener, TestQuerySenderNoQuery, CopyFieldTest, SolrIndexConfigTest, BadComponentTest, TestStressRecovery, EchoParamsTest, TestLMDirichletSimilarityFactory, TestFastWriter, ResourceLoaderTest, TestFastOutputStream, ChaosMonkeyNothingIsSafeTest, AssignTest, CdcrBootstrapTest, CdcrRequestHandlerTest, CdcrVersionReplicationTest, CleanupOldIndexTest, CloudExitableDirectoryReaderTest, CollectionStateFormat2Test, ConnectionManagerTest, CustomCollectionTest, DeleteLastCustomShardedReplicaTest, DeleteStatusTest, DistribDocExpirationUpdateProcessorTest, DistribJoinFromCollectionTest, DistributedQueueTest, DistributedVersionInfoTest, ForceLeaderTest, HttpPartitionTest, LeaderFailoverAfterPartitionTest, LeaderFailureAfterFreshStartTest, LeaderInitiatedRecoveryOnShardRestartTest, MigrateRouteKeyTest, MultiThreadedOCPTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerModifyCollectionTest, OverseerRolesTest, OverseerStatusTest, OverseerTaskQueueTest, PeerSyncReplicationTest, RecoveryAfterSoftCommitTest, RemoteQueryErrorTest, ReplaceNodeTest, ReplicationFactorTest, SSLMigrationTest, SaslZkACLProviderTest, SharedFSAutoReplicaFailoverTest, SharedFSAutoReplicaFailoverUtilsTest, SolrCLIZkUtilsTest, SolrXmlInZkTest, TestAuthenticationFramework, TestCloudDeleteByQuery, TestCloudInspectUtil, TestCloudPseudoReturnFields, TestConfigSetsAPI, TestExclusionRuleCollectionAccess, TestLocalFSCloudBackupRestore, TestRandomFlRTGCloud, TestRequestStatusCollectionAPI, TestSSLRandomization, TestSolrCloudWithKerberosAlt, TestStressCloudBlindAtomicUpdates, TriLevelCompositeIdRoutingTest, VMParamsZkACLAndCredentialsProvidersTest, HdfsBasicDistributedZk2Test, HdfsBasicDistributedZkTest, HdfsChaosMonkeyNothingIsSafeTest, HdfsChaosMonkeySafeLeaderTest, HdfsNNFailoverTest, HdfsSyncSliceTest, HdfsThreadLeakTest, HdfsUnloadDistributedZkTest, StressHdfsTest]
   [junit4] Completed [428/635 (6!)] on J0 in 88.19s, 1 test, 1 failure <<< FAILURES!

[...truncated 420 lines...]
   [junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20161005_083740_204.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid6781.hprof ...
   [junit4] Heap dump file created [588108079 bytes in 19.831 secs]
   [junit4] <<< JVM J0: EOF ----

[...truncated 7400 lines...]
ERROR: Connection was broken: java.io.IOException: Unexpected termination of the channel
	at hudson.remoting.SynchronousCommandTransport$ReaderThread.run(SynchronousCommandTransport.java:73)
Caused by: java.io.EOFException
	at java.io.ObjectInputStream$PeekInputStream.readFully(ObjectInputStream.java:2328)
	at java.io.ObjectInputStream$BlockDataInputStream.readShort(ObjectInputStream.java:2797)
	at java.io.ObjectInputStream.readStreamHeader(ObjectInputStream.java:802)
	at java.io.ObjectInputStream.<init>(ObjectInputStream.java:299)
	at hudson.remoting.ObjectInputStreamEx.<init>(ObjectInputStreamEx.java:48)
	at hudson.remoting.AbstractSynchronousByteArrayCommandTransport.read(AbstractSynchronousByteArrayCommandTransport.java:34)
	at hudson.remoting.SynchronousCommandTransport$ReaderThread.run(SynchronousCommandTransport.java:59)

Build step 'Invoke Ant' marked build as failure
ERROR: Step ‘Archive the artifacts’ failed: no workspace for Lucene-Solr-NightlyTests-master #1126
ERROR: Step ‘Publish JUnit test result report’ failed: no workspace for Lucene-Solr-NightlyTests-master #1126
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1125 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1125/

13 tests failed.
FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBufferOnNonLeader

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([AD179BA835260260]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CdcrReplicationDistributedZkTest

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([AD179BA835260260]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CdcrReplicationDistributedZkTest

Error Message:
Captured an uncaught exception in thread: Thread[id=125282, name=zkCallback-29018-thread-8, state=RUNNABLE, group=TGRP-CdcrReplicationDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=125282, name=zkCallback-29018-thread-8, state=RUNNABLE, group=TGRP-CdcrReplicationDistributedZkTest]
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
	at __randomizedtesting.SeedInfo.seed([AD179BA835260260]:0)
	at java.lang.ThreadLocal.createMap(ThreadLocal.java:244)
	at java.lang.ThreadLocal.set(ThreadLocal.java:205)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:215)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$9/79255960.run(Unknown Source)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CdcrReplicationDistributedZkTest

Error Message:
Captured an uncaught exception in thread: Thread[id=125278, name=zkCallback-29018-thread-4, state=RUNNABLE, group=TGRP-CdcrReplicationDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=125278, name=zkCallback-29018-thread-4, state=RUNNABLE, group=TGRP-CdcrReplicationDistributedZkTest]
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
	at __randomizedtesting.SeedInfo.seed([AD179BA835260260]:0)
	at java.lang.ThreadLocal.createMap(ThreadLocal.java:244)
	at java.lang.ThreadLocal.set(ThreadLocal.java:205)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:215)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$9/79255960.run(Unknown Source)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CdcrReplicationDistributedZkTest

Error Message:
Captured an uncaught exception in thread: Thread[id=121612, name=Connection evictor, state=RUNNABLE, group=TGRP-CdcrReplicationDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=121612, name=Connection evictor, state=RUNNABLE, group=TGRP-CdcrReplicationDistributedZkTest]
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.cloud.CdcrReplicationHandlerTest.testFullReplication

Error Message:
There are still nodes recoverying - waited for 330 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 330 seconds
	at __randomizedtesting.SeedInfo.seed([AD179BA835260260:D717A1E5629944E0]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:181)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:138)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:133)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForRecoveriesToFinish(BaseCdcrDistributedZkTest.java:509)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.restartServer(BaseCdcrDistributedZkTest.java:550)
	at org.apache.solr.cloud.CdcrReplicationHandlerTest.testFullReplication(CdcrReplicationHandlerTest.java:82)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrRequestHandlerTest.testBufferActions

Error Message:
Captured an uncaught exception in thread: Thread[id=83479, name=qtp428698500-83479, state=RUNNABLE, group=TGRP-CdcrRequestHandlerTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=83479, name=qtp428698500-83479, state=RUNNABLE, group=TGRP-CdcrRequestHandlerTest]
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
	at __randomizedtesting.SeedInfo.seed([AD179BA835260260]:0)
	at java.lang.Thread.start0(Native Method)
	at java.lang.Thread.start(Thread.java:714)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.startThreads(QueuedThreadPool.java:462)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.access$200(QueuedThreadPool.java:47)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:639)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
Timeout waiting for all live and active

Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
	at __randomizedtesting.SeedInfo.seed([AD179BA835260260:2543A4729BDA6F98]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:309)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:127)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
Expected 2 of 3 replicas to be active but only found 1; [core_node2:{"core":"c8n_1x3_lf_shard1_replica3","base_url":"http://127.0.0.1:51520/_uzc","node_name":"127.0.0.1:51520__uzc","state":"active","leader":"true"}]; clusterState: DocCollection(c8n_1x3_lf//collections/c8n_1x3_lf/state.json/19)={   "replicationFactor":"3",   "shards":{"shard1":{       "range":"80000000-7fffffff",       "state":"active",       "replicas":{         "core_node1":{           "state":"down",           "base_url":"http://127.0.0.1:45032/_uzc",           "core":"c8n_1x3_lf_shard1_replica1",           "node_name":"127.0.0.1:45032__uzc"},         "core_node2":{           "core":"c8n_1x3_lf_shard1_replica3",           "base_url":"http://127.0.0.1:51520/_uzc",           "node_name":"127.0.0.1:51520__uzc",           "state":"active",           "leader":"true"},         "core_node3":{           "core":"c8n_1x3_lf_shard1_replica2",           "base_url":"http://127.0.0.1:34773/_uzc",           "node_name":"127.0.0.1:34773__uzc",           "state":"down"}}}},   "router":{"name":"compositeId"},   "maxShardsPerNode":"1",   "autoAddReplicas":"false"}

Stack Trace:
java.lang.AssertionError: Expected 2 of 3 replicas to be active but only found 1; [core_node2:{"core":"c8n_1x3_lf_shard1_replica3","base_url":"http://127.0.0.1:51520/_uzc","node_name":"127.0.0.1:51520__uzc","state":"active","leader":"true"}]; clusterState: DocCollection(c8n_1x3_lf//collections/c8n_1x3_lf/state.json/19)={
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node1":{
          "state":"down",
          "base_url":"http://127.0.0.1:45032/_uzc",
          "core":"c8n_1x3_lf_shard1_replica1",
          "node_name":"127.0.0.1:45032__uzc"},
        "core_node2":{
          "core":"c8n_1x3_lf_shard1_replica3",
          "base_url":"http://127.0.0.1:51520/_uzc",
          "node_name":"127.0.0.1:51520__uzc",
          "state":"active",
          "leader":"true"},
        "core_node3":{
          "core":"c8n_1x3_lf_shard1_replica2",
          "base_url":"http://127.0.0.1:34773/_uzc",
          "node_name":"127.0.0.1:34773__uzc",
          "state":"down"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false"}
	at __randomizedtesting.SeedInfo.seed([AD179BA835260260:2543A4729BDA6F98]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:170)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:57)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)
	at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)
	at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(Sys

[...truncated too long message...]

e.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 12100615 WARN  (TEST-CdcrReplicationDistributedZkTest.testBufferOnNonLeader-seed#[AD179BA835260260]) [    ] o.a.s.c.ZkController Could not publish node as down: java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 12100616 INFO  (TEST-CdcrReplicationDistributedZkTest.testBufferOnNonLeader-seed#[AD179BA835260260]) [    ] o.a.s.c.Overseer Overseer (id=96700331920916505-127.0.0.1:53309_snxqv-n_0000000006) closing
   [junit4]   2> 12100674 INFO  (TEST-CdcrReplicationDistributedZkTest.testBufferOnNonLeader-seed#[AD179BA835260260]) [    ] o.e.j.s.ServerConnector Stopped ServerConnector@45d0ea17{SSL,[ssl, http/1.1]}{127.0.0.1:0}
   [junit4]   2> 12100674 INFO  (TEST-CdcrReplicationDistributedZkTest.testBufferOnNonLeader-seed#[AD179BA835260260]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@4158b3bb{/snxqv,null,UNAVAILABLE}
   [junit4]   2> 12100786 INFO  (TEST-CdcrReplicationDistributedZkTest.testBufferOnNonLeader-seed#[AD179BA835260260]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testBufferOnNonLeader
   [junit4]   2> 12100788 INFO  (TEST-CdcrReplicationDistributedZkTest.testBufferOnNonLeader-seed#[AD179BA835260260]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:60345 60345
   [junit4]   2> 12101120 INFO  (SUITE-CdcrReplicationDistributedZkTest-seed#[AD179BA835260260]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_AD179BA835260260-001
   [junit4]   2> Oct 03, 2016 11:28:07 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {}, docValues:{}, maxPointsInLeafNode=990, maxMBSortInHeap=7.464792411998001, sim=RandomSimilarity(queryNorm=false): {}, locale=sr-Latn-RS, timezone=Pacific/Kwajalein
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=2,free=117083504,total=493879296
   [junit4]   2> NOTE: All tests run in this JVM: [CollectionsAPIDistributedZkTest, BlockJoinFacetRandomTest, TestCoreContainer, TestFiltering, TestElisionMultitermQuery, ResponseHeaderTest, TestRandomRequestDistribution, TestExpandComponent, TestStressLiveNodes, TestSchemaSimilarityResource, TestSimpleQParserPlugin, CSVRequestHandlerTest, TestSimpleTrackingShardHandler, TestSolrCloudWithKerberosAlt, TestJsonFacets, FileUtilsTest, TestLeaderInitiatedRecoveryThread, ZkControllerTest, AlternateDirectoryTest, TestBinaryResponseWriter, SharedFSAutoReplicaFailoverUtilsTest, TestCloudDeleteByQuery, TestSolrQueryParserDefaultOperatorResource, SOLR749Test, PreAnalyzedUpdateProcessorTest, CoreMergeIndexesAdminHandlerTest, TestBulkSchemaConcurrent, TestNumericTerms32, TestDocTermOrds, MoreLikeThisHandlerTest, HdfsDirectoryFactoryTest, DistanceFunctionTest, SaslZkACLProviderTest, TestConfigSets, ClusterStateUpdateTest, TestShortCircuitedRequests, ConvertedLegacyTest, RestartWhileUpdatingTest, CollectionTooManyReplicasTest, DistributedIntervalFacetingTest, TestMiniSolrCloudClusterSSL, TermVectorComponentTest, BasicFunctionalityTest, CreateCollectionCleanupTest, SpellingQueryConverterTest, LeaderInitiatedRecoveryOnShardRestartTest, TestBM25SimilarityFactory, TestLockTree, IndexSchemaRuntimeFieldTest, AliasIntegrationTest, TestSolrFieldCacheMBean, TestDistribDocBasedVersion, IndexSchemaTest, AddBlockUpdateTest, ConcurrentDeleteAndCreateCollectionTest, TestHdfsBackupRestoreCore, TestDefaultSearchFieldResource, TestUpdate, LeaderFailoverAfterPartitionTest, SpatialFilterTest, TestManagedResource, RequestLoggingTest, TestMacros, UUIDUpdateProcessorFallbackTest, TestStressRecovery, SpellCheckCollatorTest, DirectSolrConnectionTest, DirectSolrSpellCheckerTest, TestHdfsUpdateLog, TestHdfsCloudBackupRestore, IndexBasedSpellCheckerTest, TestDistributedGrouping, HdfsUnloadDistributedZkTest, TestScoreJoinQPNoScore, HdfsChaosMonkeySafeLeaderTest, DistribDocExpirationUpdateProcessorTest, OpenCloseCoreStressTest, SolrIndexSplitterTest, OpenExchangeRatesOrgProviderTest, DocExpirationUpdateProcessorFactoryTest, TestRandomCollapseQParserPlugin, TlogReplayBufferedWhileIndexingTest, TestCursorMarkWithoutUniqueKey, JavabinLoaderTest, TestSolrConfigHandlerConcurrent, TestPostingsSolrHighlighter, SortByFunctionTest, SampleTest, TestFuzzyAnalyzedSuggestions, TestQueryUtils, OverseerTaskQueueTest, TestFieldCacheVsDocValues, QueryParsingTest, DocValuesMissingTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestFieldTypeResource, TestSolrDynamicMBean, TestCustomSort, TestFieldCacheSanityChecker, BigEndianAscendingWordDeserializerTest, SolrCmdDistributorTest, OverseerModifyCollectionTest, TestTrie, CdcrReplicationDistributedZkTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=CdcrReplicationDistributedZkTest -Dtests.seed=AD179BA835260260 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=sr-Latn-RS -Dtests.timezone=Pacific/Kwajalein -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J1 | CdcrReplicationDistributedZkTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AD179BA835260260]:0)Throwable #2: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=125282, name=zkCallback-29018-thread-8, state=RUNNABLE, group=TGRP-CdcrReplicationDistributedZkTest]
   [junit4]    > Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AD179BA835260260]:0)
   [junit4]    > 	at java.lang.ThreadLocal.createMap(ThreadLocal.java:244)
   [junit4]    > 	at java.lang.ThreadLocal.set(ThreadLocal.java:205)
   [junit4]    > 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:215)
   [junit4]    > 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$9/79255960.run(Unknown Source)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #3: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=125278, name=zkCallback-29018-thread-4, state=RUNNABLE, group=TGRP-CdcrReplicationDistributedZkTest]
   [junit4]    > Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AD179BA835260260]:0)
   [junit4]    > 	at java.lang.ThreadLocal.createMap(ThreadLocal.java:244)
   [junit4]    > 	at java.lang.ThreadLocal.set(ThreadLocal.java:205)
   [junit4]    > 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:215)
   [junit4]    > 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$9/79255960.run(Unknown Source)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #4: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=121612, name=Connection evictor, state=RUNNABLE, group=TGRP-CdcrReplicationDistributedZkTest]
   [junit4]    > Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Completed [636/636 (8!)] on J1 in 7320.25s, 1 test, 5 errors <<< FAILURES!

[...truncated 1 lines...]
   [junit4] JVM J1: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20161003_200617_507.sysout
   [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid10181.hprof ...
   [junit4] Heap dump file created [589912235 bytes in 13.697 secs]
   [junit4] <<< JVM J1: EOF ----

[...truncated 11039 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:715: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid10181.hprof

Total time: 507 minutes 52 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1124 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1124/

13 tests failed.
FAILED:  org.apache.solr.cloud.CdcrBootstrapTest.testConvertClusterToCdcrAndBootstrap

Error Message:
Document mismatch on target after sync expected:<10000> but was:<0>

Stack Trace:
java.lang.AssertionError: Document mismatch on target after sync expected:<10000> but was:<0>
	at __randomizedtesting.SeedInfo.seed([49AB02AE84B67800:9E7C2DD930E9E047]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.apache.solr.cloud.CdcrBootstrapTest.testConvertClusterToCdcrAndBootstrap(CdcrBootstrapTest.java:144)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationHandlerTest.testReplicationWithBufferedUpdates

Error Message:
Timeout while trying to assert number of documents @ source_collection

Stack Trace:
java.lang.AssertionError: Timeout while trying to assert number of documents @ source_collection
	at __randomizedtesting.SeedInfo.seed([49AB02AE84B67800:9AA252B0C125E497]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:273)
	at org.apache.solr.cloud.CdcrReplicationHandlerTest.testReplicationWithBufferedUpdates(CdcrReplicationHandlerTest.java:233)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.AssertionError: expected:<570> but was:<580>
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:264)
	... 42 more


FAILED:  org.apache.solr.cloud.CdcrReplicationHandlerTest.testPartialReplication

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
	at __randomizedtesting.SeedInfo.seed([49AB02AE84B67800:D789231EAC1EF86C]:0)
	at java.lang.Thread.start0(Native Method)
	at java.lang.Thread.start(Thread.java:714)
	at org.apache.http.impl.client.IdleConnectionEvictor.start(IdleConnectionEvictor.java:96)
	at org.apache.http.impl.client.HttpClientBuilder.build(HttpClientBuilder.java:1178)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:260)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.<init>(CloudSolrClient.java:387)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.<init>(CloudSolrClient.java:100)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$Builder.build(CloudSolrClient.java:1655)
	at org.apache.solr.SolrTestCaseJ4$CloudSolrClientBuilder.build(SolrTestCaseJ4.java:2115)
	at org.apache.solr.SolrTestCaseJ4.getCloudSolrClient(SolrTestCaseJ4.java:2147)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.createCloudClient(BaseCdcrDistributedZkTest.java:183)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForRecoveriesToFinish(BaseCdcrDistributedZkTest.java:505)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.restartServer(BaseCdcrDistributedZkTest.java:550)
	at org.apache.solr.cloud.CdcrReplicationHandlerTest.testPartialReplication(CdcrReplicationHandlerTest.java:116)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrRequestHandlerTest.testLifeCycleActions

Error Message:
Captured an uncaught exception in thread: Thread[id=67786, name=qtp1013051176-67786, state=RUNNABLE, group=TGRP-CdcrRequestHandlerTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=67786, name=qtp1013051176-67786, state=RUNNABLE, group=TGRP-CdcrRequestHandlerTest]
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
	at __randomizedtesting.SeedInfo.seed([49AB02AE84B67800]:0)
	at java.lang.Thread.start0(Native Method)
	at java.lang.Thread.start(Thread.java:714)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.startThreads(QueuedThreadPool.java:462)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.access$200(QueuedThreadPool.java:47)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:639)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CollectionStateFormat2Test.test

Error Message:
Shards in the state does not match what we set:3 vs 4

Stack Trace:
java.lang.AssertionError: Shards in the state does not match what we set:3 vs 4
	at __randomizedtesting.SeedInfo.seed([49AB02AE84B67800:C1FF3D742A4A15F8]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:419)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:332)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:983)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
Expected 2 of 3 replicas to be active but only found 1; [core_node3:{"core":"c8n_1x3_lf_shard1_replica2","base_url":"http://127.0.0.1:54428","node_name":"127.0.0.1:54428_","state":"active","leader":"true"}]; clusterState: DocCollection(c8n_1x3_lf//collections/c8n_1x3_lf/state.json/19)={   "replicationFactor":"3",   "shards":{"shard1":{       "range":"80000000-7fffffff",       "state":"active",       "replicas":{         "core_node1":{           "state":"down",           "base_url":"http://127.0.0.1:56560",           "core":"c8n_1x3_lf_shard1_replica1",           "node_name":"127.0.0.1:56560_"},         "core_node2":{           "core":"c8n_1x3_lf_shard1_replica3",           "base_url":"http://127.0.0.1:48001",           "node_name":"127.0.0.1:48001_",           "state":"down"},         "core_node3":{           "core":"c8n_1x3_lf_shard1_replica2",           "base_url":"http://127.0.0.1:54428",           "node_name":"127.0.0.1:54428_",           "state":"active",           "leader":"true"}}}},   "router":{"name":"compositeId"},   "maxShardsPerNode":"1",   "autoAddReplicas":"false"}

Stack Trace:
java.lang.AssertionError: Expected 2 of 3 replicas to be active but only found 1; [core_node3:{"core":"c8n_1x3_lf_shard1_replica2","base_url":"http://127.0.0.1:54428","node_name":"127.0.0.1:54428_","state":"active","leader":"true"}]; clusterState: DocCollection(c8n_1x3_lf//collections/c8n_1x3_lf/state.json/19)={
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node1":{
          "state":"down",
          "base_url":"http://127.0.0.1:56560",
          "core":"c8n_1x3_lf_shard1_replica1",
          "node_name":"127.0.0.1:56560_"},
        "core_node2":{
          "core":"c8n_1x3_lf_shard1_replica3",
          "base_url":"http://127.0.0.1:48001",
          "node_name":"127.0.0.1:48001_",
          "state":"down"},
        "core_node3":{
          "core":"c8n_1x3_lf_shard1_replica2",
          "base_url":"http://127.0.0.1:54428",
          "node_name":"127.0.0.1:54428_",
          "state":"active",
          "leader":"true"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false"}
	at __randomizedtesting.SeedInfo.seed([49AB02AE84B67800:C1FF3D742A4A15F8]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:170)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:57)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.update.PeerSyncTest.test

Error Message:
.response[0][id][0]:3!=7

Stack Trace:
junit.framework.AssertionFailedError: .response[0][id][0]:3!=7
	at __randomizedtesting.SeedInfo.seed([49AB02AE84B67800:C1FF3D742A4A15F8]:0)
	at junit.framework.Assert.fail(Assert.java:50)
	at org.apache.solr.BaseDistributedSearchTestCase.compareSolrResponses(BaseDistributedSearchTestCase.java:913)
	at org.apache.solr.BaseDistributedSearchTestCase.compareResponses(BaseDistributedSearchTestCase.java:932)
	at org.apache.solr.BaseDistributedSearchTestCase.queryAndCompare(BaseDistributedSearchTestCase.java:650)
	at org.apache.solr.BaseDistributedSearchTestCase.queryAndCompare(BaseDistributedSearchTestCase.java:641)
	at org.apache.solr.update.PeerSyncTest.test(PeerSyncTest.java:97)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2250)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2250)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2250)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2250)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomize

[...truncated too long message...]

2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-collective-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-event-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/bcprov-jdk15-1.45.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/cglib-nodep-2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/commons-math3-3.4.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/easymock-3.0.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-common-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-hdfs-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-minikdc-2.7.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-annotations-2.5.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-databind-2.5.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/mockito-core-1.9.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/objenesis-1.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/presto-parser-0.122.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/slice-0.10.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/analysis/icu/lucene-analyzers-icu-7.0.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/analysis-extras/lib/icu4j-56.1.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_102/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.3.4.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20161002_094651_442.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20161002_094651_442.suites -stdin
   [junit4] ERROR: JVM J2 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1501)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:121)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:955)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:952)
   [junit4] 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4] 	at java.lang.Thread.run(Thread.java:745)

BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:707: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build.xml:252: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/common-build.xml:536: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:1443: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:984: At least one slave process threw an exception, first: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.

Total time: 429 minutes 5 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1123 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1123/

16 tests failed.
FAILED:  org.apache.lucene.search.TestFuzzyQuery.testRandom

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([2D27C75D3AAA0A9]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.search.TestFuzzyQuery

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([2D27C75D3AAA0A9]:0)


FAILED:  org.apache.solr.cloud.CdcrRequestHandlerTest.testBufferActions

Error Message:
Captured an uncaught exception in thread: Thread[id=68967, name=SessionTracker, state=RUNNABLE, group=TGRP-CdcrRequestHandlerTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=68967, name=SessionTracker, state=RUNNABLE, group=TGRP-CdcrRequestHandlerTest]
	at __randomizedtesting.SeedInfo.seed([DF3C5BBDBF828ECA:4FEA04327916FAC0]:0)
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
	at __randomizedtesting.SeedInfo.seed([DF3C5BBDBF828ECA]:0)
	at java.lang.Long.valueOf(Long.java:840)
	at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:150)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CdcrVersionReplicationTest

Error Message:
8 threads leaked from SUITE scope at org.apache.solr.cloud.CdcrVersionReplicationTest:     1) Thread[id=236749, name=qtp1046926853-236749, state=TIMED_WAITING, group=TGRP-CdcrVersionReplicationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=236751, name=qtp1046926853-236751-acceptor-0@1ece7e42-ServerConnector@607df54b{SSL,[ssl, http/1.1]}{127.0.0.1:54464}, state=RUNNABLE, group=TGRP-CdcrVersionReplicationTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:382)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:593)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=236754, name=qtp1046926853-236754, state=TIMED_WAITING, group=TGRP-CdcrVersionReplicationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=236753, name=qtp1046926853-236753, state=TIMED_WAITING, group=TGRP-CdcrVersionReplicationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=236750, name=qtp1046926853-236750, state=RUNNABLE, group=TGRP-CdcrVersionReplicationTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=236755, name=qtp1046926853-236755, state=TIMED_WAITING, group=TGRP-CdcrVersionReplicationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=236752, name=qtp1046926853-236752, state=TIMED_WAITING, group=TGRP-CdcrVersionReplicationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=236748, name=qtp1046926853-236748, state=RUNNABLE, group=TGRP-CdcrVersionReplicationTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.cloud.CdcrVersionReplicationTest: 
   1) Thread[id=236749, name=qtp1046926853-236749, state=TIMED_WAITING, group=TGRP-CdcrVersionReplicationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=236751, name=qtp1046926853-236751-acceptor-0@1ece7e42-ServerConnector@607df54b{SSL,[ssl, http/1.1]}{127.0.0.1:54464}, state=RUNNABLE, group=TGRP-CdcrVersionReplicationTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:382)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:593)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=236754, name=qtp1046926853-236754, state=TIMED_WAITING, group=TGRP-CdcrVersionReplicationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=236753, name=qtp1046926853-236753, state=TIMED_WAITING, group=TGRP-CdcrVersionReplicationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=236750, name=qtp1046926853-236750, state=RUNNABLE, group=TGRP-CdcrVersionReplicationTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=236755, name=qtp1046926853-236755, state=TIMED_WAITING, group=TGRP-CdcrVersionReplicationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=236752, name=qtp1046926853-236752, state=TIMED_WAITING, group=TGRP-CdcrVersionReplicationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=236748, name=qtp1046926853-236748, state=RUNNABLE, group=TGRP-CdcrVersionReplicationTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([DF3C5BBDBF828ECA]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CdcrVersionReplicationTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=236750, name=qtp1046926853-236750, state=RUNNABLE, group=TGRP-CdcrVersionReplicationTest]         at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)         at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)         at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=236748, name=qtp1046926853-236748, state=RUNNABLE, group=TGRP-CdcrVersionReplicationTest]         at sun.nio.ch.IOUtil.drain(Native Method)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=236750, name=qtp1046926853-236750, state=RUNNABLE, group=TGRP-CdcrVersionReplicationTest]
        at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)
        at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)
        at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)
        at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
        at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=236748, name=qtp1046926853-236748, state=RUNNABLE, group=TGRP-CdcrVersionReplicationTest]
        at sun.nio.ch.IOUtil.drain(Native Method)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([DF3C5BBDBF828ECA]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2250)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2250)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2250)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2250)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2250)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)
	at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2250)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)
	at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2250)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)
	at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2250)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMar

[...truncated too long message...]

artStop-seed#[DF3C5BBDBF828ECA]) [    ] o.e.j.s.ServerConnector Stopped ServerConnector@6c2e8e32{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 7902385 INFO  (TEST-CdcrReplicationDistributedZkTest.testReplicationStartStop-seed#[DF3C5BBDBF828ECA]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@87752fc{/_vd/y,null,UNAVAILABLE}
   [junit4]   2> 7902386 INFO  (TEST-CdcrReplicationDistributedZkTest.testReplicationStartStop-seed#[DF3C5BBDBF828ECA]) [    ] o.a.s.c.ChaosMonkey monkey: stop shard! 57363
   [junit4]   2> 7902386 INFO  (TEST-CdcrReplicationDistributedZkTest.testReplicationStartStop-seed#[DF3C5BBDBF828ECA]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1872502112
   [junit4]   2> 7902392 INFO  (TEST-CdcrReplicationDistributedZkTest.testReplicationStartStop-seed#[DF3C5BBDBF828ECA]) [    ] o.a.s.c.Overseer Overseer (id=96685160968945696-127.0.0.1:57363__vd%2Fy-n_0000000007) closing
   [junit4]   2> 7902392 INFO  (OverseerStateUpdate-96685160968945696-127.0.0.1:57363__vd%2Fy-n_0000000007) [n:127.0.0.1:57363__vd%2Fy    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:57363__vd%2Fy
   [junit4]   2> 7902405 WARN  (zkCallback-3673-thread-3-processing-n:127.0.0.1:57363__vd%2Fy) [n:127.0.0.1:57363__vd%2Fy    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 7903480 WARN  (zkCallback-3673-thread-4-processing-n:127.0.0.1:57363__vd%2Fy) [n:127.0.0.1:57363__vd%2Fy c:target_collection s:shard2 r:core_node4 x:target_collection_shard2_replica2] o.a.s.c.SyncStrategy Closed, skipping sync up.
   [junit4]   2> 7903481 INFO  (zkCallback-3673-thread-4-processing-n:127.0.0.1:57363__vd%2Fy) [n:127.0.0.1:57363__vd%2Fy c:target_collection s:shard2 r:core_node4 x:target_collection_shard2_replica2] o.a.s.c.SolrCore [target_collection_shard2_replica2]  CLOSING SolrCore org.apache.solr.core.SolrCore@3676c76b
   [junit4]   2> 7903481 INFO  (zkCallback-3673-thread-4-processing-n:127.0.0.1:57363__vd%2Fy) [n:127.0.0.1:57363__vd%2Fy c:target_collection s:shard2 r:core_node4 x:target_collection_shard2_replica2] o.a.s.h.CdcrRequestHandler Solr core is being closed - shutting down CDCR handler @ target_collection:shard2
   [junit4]   2> 7903507 INFO  (TEST-CdcrReplicationDistributedZkTest.testReplicationStartStop-seed#[DF3C5BBDBF828ECA]) [    ] o.e.j.s.ServerConnector Stopped ServerConnector@6af6b1c5{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 7903508 INFO  (TEST-CdcrReplicationDistributedZkTest.testReplicationStartStop-seed#[DF3C5BBDBF828ECA]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7a326319{/_vd/y,null,UNAVAILABLE}
   [junit4]   2> 7903508 INFO  (TEST-CdcrReplicationDistributedZkTest.testReplicationStartStop-seed#[DF3C5BBDBF828ECA]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testReplicationStartStop
   [junit4]   2> 7903512 INFO  (TEST-CdcrReplicationDistributedZkTest.testReplicationStartStop-seed#[DF3C5BBDBF828ECA]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:59905 59905
   [junit4]   2> 7903649 INFO  (Thread-20183) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:59905 59905
   [junit4]   2> 7903658 WARN  (Thread-20183) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	33	/solr/clusterprops.json
   [junit4]   2> 	33	/solr/aliases.json
   [junit4]   2> 	16	/solr/configs/conf1
   [junit4]   2> 	8	/solr/collections/tmp_collection/state.json
   [junit4]   2> 	8	/solr/security.json
   [junit4]   2> 	3	/solr/collections/target_collection/leaders/shard2/leader
   [junit4]   2> 	3	/solr/collections/target_collection/leaders/shard1/leader
   [junit4]   2> 	2	/solr/collections/source_collection/leaders/shard1/leader
   [junit4]   2> 	2	/solr/collections/source_collection/leaders/shard2/leader
   [junit4]   2> 	2	/solr/collections/tmp_collection/leaders/shard2/leader
   [junit4]   2> 	2	/solr/collections/tmp_collection/leaders/shard1/leader
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	33	/solr/clusterstate.json
   [junit4]   2> 	8	/solr/collections/tmp_collection/cdcr/state/process
   [junit4]   2> 	8	/solr/collections/tmp_collection/cdcr/state/buffer
   [junit4]   2> 	7	/solr/collections/source_collection/cdcr/state/process
   [junit4]   2> 	6	/solr/collections/target_collection/cdcr/state/buffer
   [junit4]   2> 	6	/solr/collections/target_collection/cdcr/state/process
   [junit4]   2> 	4	/solr/collections/source_collection/state.json
   [junit4]   2> 	4	/solr/collections/source_collection/cdcr/state/buffer
   [junit4]   2> 	4	/solr/collections/target_collection/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	285	/solr/overseer/collection-queue-work
   [junit4]   2> 	67	/solr/overseer/queue
   [junit4]   2> 	33	/solr/collections
   [junit4]   2> 	32	/solr/live_nodes
   [junit4]   2> 	17	/solr/overseer/queue-work
   [junit4]   2> 
   [junit4]   2> 7903659 INFO  (SUITE-CdcrReplicationDistributedZkTest-seed#[DF3C5BBDBF828ECA]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrReplicationDistributedZkTest_DF3C5BBDBF828ECA-001
   [junit4]   2> Oct 01, 2016 5:13:04 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62), sim=RandomSimilarity(queryNorm=false): {}, locale=ar-MA, timezone=US/Central
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=189445672,total=500170752
   [junit4]   2> NOTE: All tests run in this JVM: [DistributedSpellCheckComponentTest, CursorPagingTest, TestSlowCompositeReaderWrapper, TestRTimerTree, TestSchemaResource, TestSimpleQParserPlugin, OverseerStatusTest, TestConfigSetsAPI, TestRecoveryHdfs, DistributedQueryElevationComponentTest, DateRangeFieldTest, FastVectorHighlighterTest, PolyFieldTest, TestLeaderElectionZkExpiry, RankQueryTest, SearchHandlerTest, OverseerCollectionConfigSetProcessorTest, DateMathParserTest, TestFieldCacheSanityChecker, TestJmxMonitoredMap, CleanupOldIndexTest, SpatialRPTFieldTypeTest, PingRequestHandlerTest, ZkNodePropsTest, ResponseHeaderTest, TestSolrCoreProperties, TestFaceting, HttpPartitionTest, QueryEqualityTest, ConnectionManagerTest, PluginInfoTest, BlockJoinFacetRandomTest, TestLFUCache, AnalysisAfterCoreReloadTest, LeaderInitiatedRecoveryOnCommitTest, TestUniqueKeyFieldResource, TestConfigReload, TestSolrCoreSnapshots, TestRemoteStreaming, TestSSLRandomization, TestRestManager, MultiThreadedOCPTest, TestFastWriter, TestFieldTypeCollectionResource, TestRandomDVFaceting, ConnectionReuseTest, BufferStoreTest, TestRandomFaceting, TestNumericTerms32, MultiTermTest, TestAuthenticationFramework, TestDFRSimilarityFactory, TestAnalyzedSuggestions, TestBackupRepositoryFactory, TestBlendedInfixSuggestions, HLLUtilTest, TestHdfsUpdateLog, TestSolrFieldCacheMBean, CollectionStateFormat2Test, DocExpirationUpdateProcessorFactoryTest, ExitableDirectoryReaderTest, HdfsNNFailoverTest, DistributedDebugComponentTest, TestFunctionQuery, TestJmxIntegration, TestCrossCoreJoin, RecoveryAfterSoftCommitTest, SSLMigrationTest, TestOmitPositions, TestMacroExpander, SignatureUpdateProcessorFactoryTest, TestJsonFacets, RequiredFieldsTest, CdcrBootstrapTest, AsyncCallRequestStatusResponseTest, TestExclusionRuleCollectionAccess, TestRawTransformer, TestSortByMinMaxFunction, PathHierarchyTokenizerFactoryTest, CacheHeaderTest, TestPKIAuthenticationPlugin, CurrencyFieldOpenExchangeTest, TestFieldCacheVsDocValues, TestSolrConfigHandler, LeaderElectionIntegrationTest, TestDynamicFieldResource, TestTrieFacet, TestStressCloudBlindAtomicUpdates, TestConfigSetProperties, TestImplicitCoreProperties, RulesTest, TestRandomFlRTGCloud, TestTolerantUpdateProcessorRandomCloud, TestGraphTermsQParserPlugin, NotRequiredUniqueKeyTest, ActionThrottleTest, TestLegacyNumericRangeQueryBuilder, TestBadConfig, SolrCLIZkUtilsTest, SmileWriterTest, SyncSliceTest, BasicZkTest, RecoveryZkTest, FullSolrCloudDistribCmdsTest, TestReplicationHandler, TestSolr4Spatial, TestGroupingSearch, HighlighterTest, SolrRequestParserTest, TestFoldingMultitermQuery, TestCSVLoader, SolrCoreCheckLockOnStartupTest, TestUpdate, FieldMutatingUpdateProcessorTest, DefaultValueUpdateProcessorTest, DocValuesMultiTest, DistanceFunctionTest, TestSolrDeletionPolicy1, XsltUpdateRequestHandlerTest, IndexBasedSpellCheckerTest, DisMaxRequestHandlerTest, DocumentAnalysisRequestHandlerTest, FileBasedSpellCheckerTest, TermVectorComponentTest, FieldAnalysisRequestHandlerTest, LoggingHandlerTest, ReturnFieldsTest, TestCSVResponseWriter, TestPartialUpdateDeduplication, TestQuerySenderListener, TestQuerySenderNoQuery, ResponseLogComponentTest, TestStressRecovery, TestHighlightDedupGrouping, AssignTest, CdcrReplicationDistributedZkTest]
   [junit4] Completed [635/635 (8!)] on J0 in 3096.27s, 11 tests, 3 errors <<< FAILURES!

[...truncated 11041 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:715: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid12170.hprof

Total time: 483 minutes 24 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1122 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1122/

9 tests failed.
FAILED:  org.apache.lucene.search.TestBooleanRewrites.testRandom

Error Message:
expected:<22> but was:<11>

Stack Trace:
java.lang.AssertionError: expected:<22> but was:<11>
	at __randomizedtesting.SeedInfo.seed([133FC438853637FC:6173E1373456818F]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.lucene.search.TestBooleanRewrites.assertEquals(TestBooleanRewrites.java:417)
	at org.apache.lucene.search.TestBooleanRewrites.testRandom(TestBooleanRewrites.java:363)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrRequestHandlerTest.testCheckpointActions

Error Message:
KeeperErrorCode = Session expired for /

Stack Trace:
org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:356)
	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:353)
	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:353)
	at org.apache.solr.common.cloud.SolrZkClient.printLayout(SolrZkClient.java:588)
	at org.apache.solr.common.cloud.SolrZkClient.printLayoutToStdOut(SolrZkClient.java:629)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForCollectionToDisappear(AbstractDistribZkTestBase.java:211)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForCollectionToDisappear(BaseCdcrDistributedZkTest.java:498)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.startServers(BaseCdcrDistributedZkTest.java:598)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.createSourceCollection(BaseCdcrDistributedZkTest.java:348)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.baseBefore(BaseCdcrDistributedZkTest.java:167)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:905)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsBasicDistributedZk2Test.test

Error Message:
Multiple exceptions

Stack Trace:
MultiException[java.lang.OutOfMemoryError: unable to create new native thread, java.util.concurrent.RejectedExecutionException: org.eclipse.jetty.io.ManagedSelector@7af7c615 id=0 keys=0 selected=0]
	at __randomizedtesting.SeedInfo.seed([D198B8B72273947A:59CC876D8C8FF982]:0)
	at org.eclipse.jetty.server.Server.doStart(Server.java:347)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:342)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:315)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:510)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:399)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:332)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:983)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
	Suppressed: java.util.concurrent.RejectedExecutionException: org.eclipse.jetty.io.ManagedSelector@7af7c615 id=0 keys=0 selected=0
		at org.eclipse.jetty.util.thread.QueuedThreadPool.execute(QueuedThreadPool.java:377)
		at org.eclipse.jetty.io.SelectorManager.execute(SelectorManager.java:125)
		at org.eclipse.jetty.io.SelectorManager.doStart(SelectorManager.java:255)
		at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
		at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:132)
		at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:106)
		at org.eclipse.jetty.server.AbstractConnector.doStart(AbstractConnector.java:260)
		at org.eclipse.jetty.server.AbstractNetworkConnector.doStart(AbstractNetworkConnector.java:81)
		at org.eclipse.jetty.server.ServerConnector.doStart(ServerConnector.java:244)
		at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
		at org.eclipse.jetty.server.Server.doStart(Server.java:384)
		... 39 more
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
	at java.lang.Thread.start0(Native Method)
	at java.lang.Thread.start(Thread.java:714)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.startThreads(QueuedThreadPool.java:462)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.doStart(QueuedThreadPool.java:113)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:132)
	at org.eclipse.jetty.server.Server.start(Server.java:405)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:106)
	at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:61)
	at org.eclipse.jetty.server.Server.doStart(Server.java:372)
	... 39 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsBasicDistributedZk2Test

Error Message:
34 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsBasicDistributedZk2Test:     1) Thread[id=2887, name=watches-439-thread-2, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=2903, name=qtp1367269790-2903, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=2916, name=zkCallback-438-thread-4, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=2913, name=OverseerStateUpdate-96675738331054089-127.0.0.1:36896_-n_0000000001, state=TIMED_WAITING, group=Overseer state updater.]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:138)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:182)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=2864, name=qtp1754747651-2864, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=2861, name=qtp1754747651-2861, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=2882, name=zkCallback-438-thread-3, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=2878, name=TEST-HdfsBasicDistributedZk2Test.test-seed#[D198B8B72273947A]-EventThread, state=WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    9) Thread[id=2880, name=zkCallback-438-thread-2, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=2865, name=qtp1754747651-2865, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   11) Thread[id=2886, name=watches-439-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   12) Thread[id=2866, name=qtp1754747651-2866, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   13) Thread[id=2905, name=qtp1367269790-2905, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   14) Thread[id=2914, name=OverseerCollectionConfigSetProcessor-96675738331054089-127.0.0.1:36896_-n_0000000001, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:336)         at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:299)         at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:279)         at org.apache.solr.cloud.DistributedQueue.peekElements(DistributedQueue.java:328)         at org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:227)         at java.lang.Thread.run(Thread.java:745)   15) Thread[id=2904, name=qtp1367269790-2904, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   16) Thread[id=2906, name=qtp1367269790-2906, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   17) Thread[id=2902, name=qtp1367269790-2902, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   18) Thread[id=2915, name=OverseerHdfsCoreFailoverThread-96675738331054089-127.0.0.1:36896_-n_0000000001, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(Thread.java:745)   19) Thread[id=2885, name=coreContainerWorkExecutor-897-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   20) Thread[id=2873, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   21) Thread[id=2877, name=TEST-HdfsBasicDistributedZk2Test.test-seed#[D198B8B72273947A]-SendThread(127.0.0.1:36395), state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)   22) Thread[id=2872, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   23) Thread[id=2899, name=coreZkRegister-896-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   24) Thread[id=2879, name=zkCallback-438-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   25) Thread[id=2862, name=qtp1754747651-2862-acceptor-0@ffe8692-ServerConnector@5884ee91{HTTP/1.1,[http/1.1]}{127.0.0.1:36896}, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:382)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:593)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)   26) Thread[id=2860, name=qtp1754747651-2860, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)   27) Thread[id=2895, name=searcherExecutor-902-thread-1, state=WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   28) Thread[id=2863, name=qtp1754747651-2863, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   29) Thread[id=2884, name=Thread-1243, state=WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1296)   30) Thread[id=2901, name=qtp1367269790-2901, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   31) Thread[id=2867, name=qtp1754747651-2867, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   32) Thread[id=2900, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   33) Thread[id=2917, name=zkCallback-438-thread-5, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   34) Thread[id=2868, name=org.eclipse.jetty.server.session.HashSessionManager@66792d02Timer, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 34 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsBasicDistributedZk2Test: 
   1) Thread[id=2887, name=watches-439-thread-2, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=2903, name=qtp1367269790-2903, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=2916, name=zkCallback-438-thread-4, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=2913, name=OverseerStateUpdate-96675738331054089-127.0.0.1:36896_-n_0000000001, state=TIMED_WAITING, group=Overseer state updater.]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:138)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:182)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=2864, name=qtp1754747651-2864, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=2861, name=qtp1754747651-2861, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=2882, name=zkCallback-438-thread-3, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=2878, name=TEST-HdfsBasicDistributedZk2Test.test-seed#[D198B8B72273947A]-EventThread, state=WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   9) Thread[id=2880, name=zkCallback-438-thread-2, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=2865, name=qtp1754747651-2865, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  11) Thread[id=2886, name=watches-439-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  12) Thread[id=2866, name=qtp1754747651-2866, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  13) Thread[id=2905, name=qtp1367269790-2905, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  14) Thread[id=2914, name=OverseerCollectionConfigSetProcessor-96675738331054089-127.0.0.1:36896_-n_0000000001, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:336)
        at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:299)
        at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:279)
        at org.apache.solr.cloud.DistributedQueue.peekElements(DistributedQueue.java:328)
        at org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:227)
        at java.lang.Thread.run(Thread.java:745)
  15) Thread[id=2904, name=qtp1367269790-2904, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  16) Thread[id=2906, name=qtp1367269790-2906, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  17) Thread[id=2902, name=qtp1367269790-2902, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  18) Thread[id=2915, name=OverseerHdfsCoreFailoverThread-96675738331054089-127.0.0.1:36896_-n_0000000001, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(Thread.java:745)
  19) Thread[id=2885, name=coreContainerWorkExecutor-897-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  20) Thread[id=2873, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  21) Thread[id=2877, name=TEST-HdfsBasicDistributedZk2Test.test-seed#[D198B8B72273947A]-SendThread(127.0.0.1:36395), state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
  22) Thread[id=2872, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  23) Thread[id=2899, name=coreZkRegister-896-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  24) Thread[id=2879, name=zkCallback-438-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  25) Thread[id=2862, name=qtp1754747651-2862-acceptor-0@ffe8692-ServerConnector@5884ee91{HTTP/1.1,[http/1.1]}{127.0.0.1:36896}, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:382)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:593)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
  26) Thread[id=2860, name=qtp1754747651-2860, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
  27) Thread[id=2895, name=searcherExecutor-902-thread-1, state=WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  28) Thread[id=2863, name=qtp1754747651-2863, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  29) Thread[id=2884, name=Thread-1243, state=WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1296)
  30) Thread[id=2901, name=qtp1367269790-2901, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  31) Thread[id=2867, name=qtp1754747651-2867, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  32) Thread[id=2900, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  33) Thread[id=2917, name=zkCallback-438-thread-5, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  34) Thread[id=2868, name=org.eclipse.jetty.server.session.HashSessionManager@66792d02Timer, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([D198B8B72273947A]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsBasicDistributedZk2Test

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=2887, name=watches-439-thread-2, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=2915, name=OverseerHdfsCoreFailoverThread-96675738331054089-127.0.0.1:36896_-n_0000000001, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.currentThread(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=2916, name=zkCallback-438-thread-4, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=2885, name=coreContainerWorkExecutor-897-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=2877, name=TEST-HdfsBasicDistributedZk2Test.test-seed#[D198B8B72273947A]-SendThread(127.0.0.1:36395), state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    6) Thread[id=2861, name=qtp1754747651-2861, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=2882, name=zkCallback-438-thread-3, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=2880, name=zkCallback-438-thread-2, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=2899, name=coreZkRegister-896-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=2879, name=zkCallback-438-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   11) Thread[id=2886, name=watches-439-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   12) Thread[id=2860, name=qtp1754747651-2860, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZk2Test]         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)   13) Thread[id=2895, name=searcherExecutor-902-thread-1, state=WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   14) Thread[id=2884, name=Thread-1243, state=WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1296)   15) Thread[id=2917, name=zkCallback-438-thread-5, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   16) Thread[id=2868, name=org.eclipse.jetty.server.session.HashSessionManager@66792d02Timer, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=2887, name=watches-439-thread-2, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=2915, name=OverseerHdfsCoreFailoverThread-96675738331054089-127.0.0.1:36896_-n_0000000001, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.currentThread(Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=2916, name=zkCallback-438-thread-4, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=2885, name=coreContainerWorkExecutor-897-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=2877, name=TEST-HdfsBasicDistributedZk2Test.test-seed#[D198B8B72273947A]-SendThread(127.0.0.1:36395), state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   6) Thread[id=2861, name=qtp1754747651-2861, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=2882, name=zkCallback-438-thread-3, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=2880, name=zkCallback-438-thread-2, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=2899, name=coreZkRegister-896-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=2879, name=zkCallback-438-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  11) Thread[id=2886, name=watches-439-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  12) Thread[id=2860, name=qtp1754747651-2860, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZk2Test]
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
  13) Thread[id=2895, name=searcherExecutor-902-thread-1, state=WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe

[...truncated too long message...]

3) Uninverting(_3(7.0.0):c1) Uninverting(_4(7.0.0):c1)))}
   [junit4]   2> 2504820 INFO  (SUITE-TestSortingResponseWriter-seed#[D198B8B72273947A]-worker) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2504820 INFO  (SUITE-TestSortingResponseWriter-seed#[D198B8B72273947A]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{commit=} 0 57
   [junit4]   2> 2504843 INFO  (TEST-TestSortingResponseWriter.testSortingOutput-seed#[D198B8B72273947A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testSortingOutput
   [junit4]   2> 2504851 INFO  (TEST-TestSortingResponseWriter.testSortingOutput-seed#[D198B8B72273947A]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:1&qt=/export&fl=floatdv,intdv,stringdv,longdv,doubledv&sort=intdv+asc&wt=xml} hits=1 status=0 QTime=7
   [junit4]   2> 2504897 INFO  (TEST-TestSortingResponseWriter.testSortingOutput-seed#[D198B8B72273947A]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:7&qt=/export&fl=floatdv,intdv,stringdv,longdv,doubledv&sort=intdv+asc&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 2504937 INFO  (TEST-TestSortingResponseWriter.testSortingOutput-seed#[D198B8B72273947A]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:1&qt=/export&fl=intdv_m,floatdv_m,doubledv_m,longdv_m,stringdv_m&sort=intdv+asc&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 2504957 INFO  (TEST-TestSortingResponseWriter.testSortingOutput-seed#[D198B8B72273947A]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:7&qt=/export&fl=intdv_m,floatdv_m,doubledv_m,longdv_m,stringdv_m&sort=intdv+asc&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 2504984 INFO  (TEST-TestSortingResponseWriter.testSortingOutput-seed#[D198B8B72273947A]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:(1+2)&qt=/export&fl=intdv&sort=intdv+desc&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 2505170 INFO  (TEST-TestSortingResponseWriter.testSortingOutput-seed#[D198B8B72273947A]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:(1+2)&qt=/export&fl=intdv&sort=intdv+asc&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 2505177 INFO  (TEST-TestSortingResponseWriter.testSortingOutput-seed#[D198B8B72273947A]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:(1+7)&qt=/export&fl=intdv&sort=stringdv+desc&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 2505274 INFO  (TEST-TestSortingResponseWriter.testSortingOutput-seed#[D198B8B72273947A]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:(1+7)&qt=/export&fl=intdv&sort=stringdv+asc&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 2505349 INFO  (TEST-TestSortingResponseWriter.testSortingOutput-seed#[D198B8B72273947A]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:(1+2)&qt=/export&fl=intdv&sort=floatdv+asc,intdv+desc&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 2505392 INFO  (TEST-TestSortingResponseWriter.testSortingOutput-seed#[D198B8B72273947A]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:(1+2)&qt=/export&fl=intdv&sort=floatdv+desc,intdv+asc&wt=xml} hits=2 status=0 QTime=2
   [junit4]   2> 2505456 INFO  (TEST-TestSortingResponseWriter.testSortingOutput-seed#[D198B8B72273947A]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:(1+2+3)&qt=/export&fl=intdv&sort=floatdv+asc,stringdv+asc,intdv+desc&wt=xml} hits=3 status=0 QTime=0
   [junit4]   2> 2505473 INFO  (TEST-TestSortingResponseWriter.testSortingOutput-seed#[D198B8B72273947A]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testSortingOutput
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSortingResponseWriter -Dtests.method=testSortingOutput -Dtests.seed=D198B8B72273947A -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=lt -Dtests.timezone=Africa/Lubumbashi -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] FAILURE 0.65s J0 | TestSortingResponseWriter.testSortingOutput <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: docs were sent out-of-order: lastDocID=2 vs doc=0
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D198B8B72273947A:65682FD62F3C9EDA]:0)
   [junit4]    > 	at org.apache.solr.response.SortingResponseWriter$FloatValue.setCurrentValue(SortingResponseWriter.java:1061)
   [junit4]    > 	at org.apache.solr.response.SortingResponseWriter$TripleValueSortDoc.setValues(SortingResponseWriter.java:679)
   [junit4]    > 	at org.apache.solr.response.SortingResponseWriter.write(SortingResponseWriter.java:179)
   [junit4]    > 	at org.apache.solr.util.TestHarness.query(TestHarness.java:316)
   [junit4]    > 	at org.apache.solr.util.TestHarness.query(TestHarness.java:292)
   [junit4]    > 	at org.apache.solr.response.TestSortingResponseWriter.testSortingOutput(TestSortingResponseWriter.java:151)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2505474 INFO  (SUITE-TestSortingResponseWriter-seed#[D198B8B72273947A]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> 2505474 INFO  (SUITE-TestSortingResponseWriter-seed#[D198B8B72273947A]-worker) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=41599093
   [junit4]   2> 2505499 INFO  (coreCloseExecutor-4340-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@2d5bdde4
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.response.TestSortingResponseWriter_D198B8B72273947A-001
   [junit4]   2> NOTE: test params are: codec=Lucene62, sim=ClassicSimilarity, locale=lt, timezone=Africa/Lubumbashi
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=220016888,total=511705088
   [junit4]   2> NOTE: All tests run in this JVM: [GraphQueryTest, RuleEngineTest, DefaultValueUpdateProcessorTest, TestJmxMonitoredMap, TestJoin, TestLazyCores, RegexBoostProcessorTest, AnalyticsMergeStrategyTest, DateMathParserTest, TestBadConfig, DirectoryFactoryTest, CdcrRequestHandlerTest, StandardRequestHandlerTest, TestAuthorizationFramework, TestFoldingMultitermQuery, HdfsRecoveryZkTest, DistributedSpellCheckComponentTest, TestCodecSupport, PeerSyncTest, MinimalSchemaTest, DistributedQueryComponentCustomSortTest, SolrCLIZkUtilsTest, TestTolerantUpdateProcessorCloud, TestOmitPositions, FastVectorHighlighterTest, DistributedFacetExistsSmallTest, TestCryptoKeys, TestPivotHelperCode, TestUninvertingReader, ZkStateReaderTest, TestPseudoReturnFields, TestLeaderInitiatedRecoveryThread, CheckHdfsIndexTest, TestTestInjection, TestSha256AuthenticationProvider, TestComplexPhraseQParserPlugin, TestCSVLoader, ZkNodePropsTest, TestLFUCache, OverseerStatusTest, TestFieldCacheReopen, TestAddFieldRealTimeGet, TestManagedSynonymFilterFactory, DeleteReplicaTest, CoreAdminCreateDiscoverTest, PathHierarchyTokenizerFactoryTest, TestClassicSimilarityFactory, SuggestComponentTest, SystemInfoHandlerTest, TestInitQParser, ReplicationFactorTest, TolerantUpdateProcessorTest, TestFunctionQuery, ReturnFieldsTest, TestRequestStatusCollectionAPI, TestAnalyzedSuggestions, DisMaxRequestHandlerTest, UUIDFieldTest, TestSolrCoreProperties, TestCloudPivotFacet, NumericFieldsTest, TestRebalanceLeaders, BasicZkTest, XsltUpdateRequestHandlerTest, DistribCursorPagingTest, BitVectorTest, TestShardHandlerFactory, TestExclusionRuleCollectionAccess, SyncSliceTest, FacetPivotSmallTest, ConnectionManagerTest, TestFieldCacheSortRandom, MergeStrategyTest, TestCollationFieldDocValues, TestAuthenticationFramework, TestRangeQuery, QueryEqualityTest, TestReplicationHandlerBackup, LeaderElectionIntegrationTest, InfoHandlerTest, SolrRequestParserTest, SharedFSAutoReplicaFailoverTest, TestIntervalFaceting, CdcrBootstrapTest, DeleteInactiveReplicaTest, QueryElevationComponentTest, CloneFieldUpdateProcessorFactoryTest, TestConfigSetsAPI, ChangedSchemaMergeTest, TestDFRSimilarityFactory, OverseerCollectionConfigSetProcessorTest, TestSuggestSpellingConverter, TestReRankQParserPlugin, DistributedQueueTest, HdfsLockFactoryTest, CleanupOldIndexTest, AnalysisAfterCoreReloadTest, LeaderInitiatedRecoveryOnCommitTest, TestOrdValues, TestSortingResponseWriter]
   [junit4] Completed [633/633 (4!)] on J0 in 1.29s, 1 test, 1 failure <<< FAILURES!

[...truncated 1 lines...]
   [junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20160929_130427_689.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid12261.hprof ...
   [junit4] Heap dump file created [533377174 bytes in 9.381 secs]
   [junit4] <<< JVM J0: EOF ----

[...truncated 11038 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:715: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid12261.hprof

Total time: 324 minutes 52 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1121 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1121/

No tests ran.

Build Log:
[...truncated 1895 lines...]
   [junit4] Suite: org.apache.lucene.search.TestFuzzyQuery
   [junit4]   2> 9월 25, 2016 4:08:54 오전 com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.search.TestFuzzyQuery
   [junit4]   2>    1) Thread[id=2972, name=SUITE-TestFuzzyQuery-seed#[AEF39117F58BD628], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1556)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:688)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:685)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:685)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:701)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:681)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:64)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:414)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:681)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:140)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:591)
   [junit4]   2>    2) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1249)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1323)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:601)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:450)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:243)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:354)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:10)
   [junit4]   2>    3) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:47)
   [junit4]   2>    4) Thread[id=2973, name=TEST-TestFuzzyQuery.testRandom-seed#[AEF39117F58BD628], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:254)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:151)
   [junit4]   2>         at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:730)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.randomSimpleString(TestFuzzyQuery.java:506)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.testRandom(TestFuzzyQuery.java:517)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestFuzzyQuery -Dtests.method=testRandom -Dtests.seed=AEF39117F58BD628 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=ko-KR -Dtests.timezone=Africa/Mogadishu -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   7200s J2 | TestFuzzyQuery.testRandom <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AEF39117F58BD628]:0)
   [junit4]   2> 9월 25, 2016 4:08:54 오전 com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> 9월 25, 2016 4:09:14 오전 com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.search.TestFuzzyQuery: 
   [junit4]   2>    1) Thread[id=2973, name=TEST-TestFuzzyQuery.testRandom-seed#[AEF39117F58BD628], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:151)
   [junit4]   2>         at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:730)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.randomSimpleString(TestFuzzyQuery.java:506)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.testRandom(TestFuzzyQuery.java:517)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 9월 25, 2016 4:09:14 오전 com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=2973, name=TEST-TestFuzzyQuery.testRandom-seed#[AEF39117F58BD628], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2> 9월 25, 2016 4:09:17 오전 com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=2973, name=TEST-TestFuzzyQuery.testRandom-seed#[AEF39117F58BD628], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.getRandomness(RandomizedContext.java:110)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.getRandom(RandomizedContext.java:132)
   [junit4]   2>         at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:730)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.randomSimpleString(TestFuzzyQuery.java:506)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.testRandom(TestFuzzyQuery.java:517)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {field=PostingsFormat(name=Memory doPackFST= false)}, docValues:{}, maxPointsInLeafNode=1489, maxMBSortInHeap=7.472815105601821, sim=RandomSimilarity(queryNorm=true): {field=DFR I(F)B3(800.0)}, locale=ko-KR, timezone=Africa/Mogadishu
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=2,free=276095768,total=426770432
   [junit4]   2> NOTE: All tests run in this JVM: [TestParallelReaderEmptyIndex, TestSingleInstanceLockFactory, TestBlockPostingsFormat, TestStressIndexing, TestAtomicUpdate, TestIndexWriterDeleteByQuery, TestBytesRefArray, TestToken, TestCodecHoldsOpenFiles, Test2BBinaryDocValues, TestCachingCollector, TestDocumentWriter, TestIndexWriterMergePolicy, TestArrayUtil, TestNot, TestNotDocIdSet, TestRegexpRandom2, TestIndexInput, TestInPlaceMergeSorter, TestTermsEnum, TestFilterWeight, TestSpanFirstQuery, TestSameScoresWithThreads, TestCollectionUtil, TestConjunctionDISI, TestIndexWriterExceptions, TestSparseFixedBitSet, TestMatchNoDocsQuery, TestUpgradeIndexMergePolicy, TestBinaryTerms, TestSimpleSearchEquivalence, TestReaderWrapperDVTypeCheck, TestNoMergeScheduler, TestSegmentTermDocs, TestIndexWriterReader, TestNRTCachingDirectory, TestSegmentMerger, TestTopDocsMerge, TestTerms, TestRegexpRandom, TestIntroSorter, TestDocIdsWriter, TestTermQuery, TestFixedBitDocIdSet, TestDirectMonotonic, TestTransactionRollback, TestIntsRef, TestTransactions, TestVirtualMethod, TestIndexWriterThreadsToSegments, TestHighCompressionMode, TestLogMergePolicy, TestSpanCollection, TestBlendedTermQuery, TestPointQueries, TestGeoUtils, LimitedFiniteStringsIteratorTest, TestParallelTermEnum, TestSnapshotDeletionPolicy, TestMergeSchedulerExternal, TestIndexWriterMaxDocs, TestOrdinalMap, TestWordlistLoader, TestDocInverterPerFieldErrorInfo, TestSpans, TestRateLimiter, TestTwoPhaseCommitTool, TestDocValuesScoring, TestLazyProxSkipping, TestMultiTermsEnum, TestWildcard, TestIndexWriterMerging, TestFSTs, TestMultiMMap, TestDeletionPolicy, TestFlushByRamOrCountsPolicy, TestIndexWriterDelete, TestNRTThreads, TestMultiThreadTermVectors, TestSort, TestMultiDocValues, TestSpanExplanations, TestStressIndexing2, TestCodecs, TestParallelLeafReader, TestSloppyPhraseQuery2, TestFuzzyQuery]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestFuzzyQuery -Dtests.seed=AEF39117F58BD628 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=ko-KR -Dtests.timezone=Africa/Mogadishu -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J2 | TestFuzzyQuery (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AEF39117F58BD628]:0)
   [junit4] Completed [433/433 (1!)] on J2 in 7223.45s, 11 tests, 2 errors <<< FAILURES!

[...truncated 9384 lines...]
   [junit4] Suite: org.apache.solr.update.SoftAutoCommitTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.update.SoftAutoCommitTest_BF4DC0948A35D7ED-001/init-core-data-001
   [junit4]   2> 13712 INFO  (SUITE-SoftAutoCommitTest-seed#[BF4DC0948A35D7ED]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 13713 INFO  (SUITE-SoftAutoCommitTest-seed#[BF4DC0948A35D7ED]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore
   [junit4]   2> 13714 INFO  (SUITE-SoftAutoCommitTest-seed#[BF4DC0948A35D7ED]-worker) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 14233 WARN  (SUITE-SoftAutoCommitTest-seed#[BF4DC0948A35D7ED]-worker) [    ] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 14256 INFO  (SUITE-SoftAutoCommitTest-seed#[BF4DC0948A35D7ED]-worker) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 14628 INFO  (SUITE-SoftAutoCommitTest-seed#[BF4DC0948A35D7ED]-worker) [    ] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 14842 INFO  (SUITE-SoftAutoCommitTest-seed#[BF4DC0948A35D7ED]-worker) [    ] o.a.s.s.IndexSchema [null] Schema name=test
   [junit4]   2> 16984 INFO  (SUITE-SoftAutoCommitTest-seed#[BF4DC0948A35D7ED]-worker) [    ] o.a.s.s.IndexSchema [null] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 16986 INFO  (SUITE-SoftAutoCommitTest-seed#[BF4DC0948A35D7ED]-worker) [    ] o.a.s.s.IndexSchema [null] unique key field: id
   [junit4]   2> 17407 INFO  (coreLoadExecutor-18-thread-1) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 17567 WARN  (coreLoadExecutor-18-thread-1) [    ] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 17569 INFO  (coreLoadExecutor-18-thread-1) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 17685 INFO  (coreLoadExecutor-18-thread-1) [    ] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 18076 INFO  (coreLoadExecutor-18-thread-1) [    ] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 18406 INFO  (coreLoadExecutor-18-thread-1) [    ] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 18407 INFO  (coreLoadExecutor-18-thread-1) [    ] o.a.s.s.IndexSchema [collection1] unique key field: id
   [junit4]   2> 18540 INFO  (coreLoadExecutor-18-thread-1) [    ] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1
   [junit4]   2> 18541 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.update.SoftAutoCommitTest_BF4DC0948A35D7ED-001/init-core-data-001/]
   [junit4]   2> 18561 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.SolrCore New index directory detected: old=null new=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.update.SoftAutoCommitTest_BF4DC0948A35D7ED-001/init-core-data-001/index/
   [junit4]   2> 18562 WARN  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1] Solr index directory '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.update.SoftAutoCommitTest_BF4DC0948A35D7ED-001/init-core-data-001/index' doesn't exist. Creating new index...
   [junit4]   2> 18562 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=29, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 18581 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@1ccd07a0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@417b113),segFN=segments_1,generation=1}
   [junit4]   2> 18581 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 18589 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 1 plugins of type queryResponseWriter: [xml]
   [junit4]   2> 18618 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 1 plugins of type queryParser: [foo]
   [junit4]   2> 18755 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 4 plugins of type searchComponent: [spellcheck, termsComp, tvComponent, highlight]
   [junit4]   2> 18755 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 18786 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 18802 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 18802 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 18802 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 18803 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 18803 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 18807 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 18807 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 18851 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 18857 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 19028 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 39 plugins of type requestHandler: [/update, /update/json, /update/csv, /update/json/docs, /config, /schema, /replication, /get, /admin/ping, /admin/segments, /admin/luke, /admin/system, /admin/mbeans, /admin/plugins, /admin/threads, /admin/properties, /admin/logging, /admin/file, /export, /graph, /stream, /sql, /terms, standard, dismax, mock, defaults, lazy, spellCheckCompRH, spellCheckCompRH_Direct, spellCheckWithWordbreak, spellCheckWithWordbreak_Direct, spellCheckCompRH1, mltrh, tvrh, /mlt, /debug/dump, /search-facet-def, /search-facet-invariants]
   [junit4]   2> 19028 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,standard,spellCheckCompRH,lazy,/update/json/docs,/admin/luke,/export,spellCheckWithWordbreak_Direct,tvrh,/get,/admin/properties,/update/json,mltrh,/admin/threads,/search-facet-def,/update/csv,/sql,/graph,/search-facet-invariants,/admin/segments,spellCheckCompRH1,/mlt,/admin/system,/replication,dismax,defaults,/config,/stream,/schema,/admin/plugins,/admin/logging,spellCheckWithWordbreak,mock,/admin/ping,/update,/admin/file,/terms,spellCheckCompRH_Direct,/debug/dump
   [junit4]   2> 19046 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 19046 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 19067 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 19068 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 19068 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=45, maxMergeAtOnceExplicit=45, maxMergedSegmentMB=97.3837890625, floorSegmentMB=1.2412109375, forceMergeDeletesPctAllowed=25.59226891409414, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8687551454356546
   [junit4]   2> 19070 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@1ccd07a0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@417b113),segFN=segments_1,generation=1}
   [junit4]   2> 19071 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 19072 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@46d7e4b[collection1] main]
   [junit4]   2> 19081 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.r.RestManager Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 19082 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 19083 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 19083 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 19246 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 19468 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 19470 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 19474 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 19474 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 19474 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 19475 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 19475 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 19476 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 19476 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 19477 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 19477 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@46d7e4b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 19480 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.UpdateLog Looking up max value of version field to seed version buckets
   [junit4]   2> 19480 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.VersionInfo Refreshing highest value of _version_ for 65536 version buckets from index
   [junit4]   2> 19480 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.VersionInfo No terms found for _version_, cannot seed version bucket highest value from index
   [junit4]   2> 19480 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1546413749740503040
   [junit4]   2> 19506 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.UpdateLog Took 25.0ms to seed version buckets with highest version 1546413749740503040
   [junit4]   2> 19509 INFO  (SUITE-SoftAutoCommitTest-seed#[BF4DC0948A35D7ED]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore end
   [junit4]   2> 19512 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testSoftAndHardCommitMaxTimeDelete
   [junit4]   2> 19513 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 20046 WARN  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    ] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 20055 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 20123 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    ] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 20213 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    ] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 21003 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    ] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 21004 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    ] o.a.s.s.IndexSchema [collection1] unique key field: id
   [junit4]   2> 21118 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    ] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1
   [junit4]   2> 21119 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.update.SoftAutoCommitTest_BF4DC0948A35D7ED-001/init-core-data-001/]
   [junit4]   2> 21120 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrCore New index directory detected: old=null new=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.update.SoftAutoCommitTest_BF4DC0948A35D7ED-001/init-core-data-001/index/
   [junit4]   2> 21121 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 1 plugins of type queryResponseWriter: [xml]
   [junit4]   2> 21124 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 1 plugins of type queryParser: [foo]
   [junit4]   2> 21132 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 4 plugins of type searchComponent: [spellcheck, termsComp, tvComponent, highlight]
   [junit4]   2> 21133 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 21149 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 21149 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 21149 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 21150 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 21163 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 21163 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 21184 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 21184 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 21184 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 21185 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 21217 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 39 plugins of type requestHandler: [/update, /update/json, /update/csv, /update/json/docs, /config, /schema, /replication, /get, /admin/ping, /admin/segments, /admin/luke, /admin/system, /admin/mbeans, /admin/plugins, /admin/threads, /admin/properties, /admin/logging, /admin/file, /export, /graph, /stream, /sql, /terms, standard, dismax, mock, defaults, lazy, spellCheckCompRH, spellCheckCompRH_Direct, spellCheckWithWordbreak, spellCheckWithWordbreak_Direct, spellCheckCompRH1, mltrh, tvrh, /mlt, /debug/dump, /search-facet-def, /search-facet-invariants]
   [junit4]   2> 21218 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,standard,spellCheckCompRH,lazy,/update/json/docs,/admin/luke,/export,spellCheckWithWordbreak_Direct,tvrh,/get,/admin/properties,/update/json,mltrh,/admin/threads,/search-facet-def,/update/csv,/sql,/graph,/search-facet-invariants,/admin/segments,spellCheckCompRH1,/mlt,/admin/system,/replication,dismax,defaults,/config,/stream,/schema,/admin/plugins,/admin/logging,spellCheckWithWordbreak,mock,/admin/ping,/update,/admin/file,/terms,spellCheckCompRH_Direct,/debug/dump
   [junit4]   2> 21224 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 21224 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 21232 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4e9d835[collection1] main]
   [junit4]   2> 21233 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.r.RestManager Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 21233 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 21233 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 21234 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 21240 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 21261 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 21261 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 21262 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 21262 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 21262 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 21262 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 21263 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 21263 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 21263 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 21264 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 21264 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@4e9d835[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 21264 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.UpdateLog Looking up max value of version field to seed version buckets
   [junit4]   2> 21265 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.VersionInfo Refreshing highest value of _version_ for 65536 version buckets from index
   [junit4]   2> 21265 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.VersionInfo No terms found for _version_, cannot seed version bucket highest value from index
   [junit4]   2> 21265 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1546413751612211200
   [junit4]   2> 21268 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.UpdateLog Took 3.0ms to seed version buckets with highest version 1546413751612211200
   [junit4]   2> 21271 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.DefaultSolrCoreState Closing old IndexWriter... core=collection1
   [junit4]   2> 21288 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@2e8d29de
   [junit4]   2> 21289 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@1ccd07a0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@417b113),segFN=segments_1,generation=1}
   [junit4]   2> 21289 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 21289 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 21290 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4b49546e[collection1] main]
   [junit4]   2> 21290 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@4b49546e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 21291 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@5c4c6b29
   [junit4]   2> 21291 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.DirectUpdateHandler2 closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=0,transaction_logs_total_number=0}
   [junit4]   2> 21291 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrCore [collection1] Closing main searcher on request.
   [junit4]   2> 21560 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[529 (1546413751642619904)]} 0 267
   [junit4]   2> 21615 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 21615 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@3b468ce5
   [junit4]   2> 21744 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@1ccd07a0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@417b113),segFN=segments_1,generation=1}
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@1ccd07a0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@417b113),segFN=segments_2,generation=2}
   [junit4]   2> 21744 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 2
   [junit4]   2> 21751 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6474949a[collection1] main]
   [junit4]   2> 21751 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 21751 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6474949a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.0.0):c1)))}
   [junit4]   2> 21776 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{commit=} 0 161
   [junit4]   2> 21785 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{delete=[529 (-1546413752149082112)]} 0 7
   [junit4]   2> 22285 INFO  (commitScheduler-27-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 22332 INFO  (commitScheduler-27-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5e7985b7[collection1] main]
   [junit4]   2> 22333 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@5e7985b7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 22338 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[550 (1546413752734187520)]} 0 3
   [junit4]   2> 22345 INFO  (commitScheduler-27-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 22844 INFO  (commitScheduler-27-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 22996 INFO  (commitScheduler-26-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 23036 INFO  (commitScheduler-27-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@72c4be05[collection1] main]
   [junit4]   2> 23038 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@72c4be05[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(7.0.0):c1)))}
   [junit4]   2> 23038 INFO  (commitScheduler-26-thread-1) [    x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@3b468ce5
   [junit4]   2> 23039 INFO  (commitScheduler-26-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@1ccd07a0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@417b113),segFN=segments_2,generation=2}
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@1ccd07a0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@417b113),segFN=segments_3,generation=3}
   [junit4]   2> 23039 INFO  (commitScheduler-26-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 3
   [junit4]   2> 23040 INFO  (commitScheduler-26-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@183a7b14[collection1] main]
   [junit4]   2> 23040 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@183a7b14[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(7.0.0):c1)))}
   [junit4]   2> 23041 INFO  (commitScheduler-26-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 23044 INFO  (commitScheduler-27-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 25040 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.SolrTestCaseJ4 ###Ending testSoftAndHardCommitMaxTimeDelete
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SoftAutoCommitTest -Dtests.method=testSoftAndHardCommitMaxTimeDelete -Dtests.seed=BF4DC0948A35D7ED -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=ar-IQ -Dtests.timezone=Etc/GMT-8 -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 5.55s J1 | SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Got a soft commit we weren't expecting
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BF4DC0948A35D7ED:78017809919D1A5D]:0)
   [junit4]    > 	at org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete(SoftAutoCommitTest.java:287)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 25068 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.SolrTestCaseJ4 ###Starting testSoftAndHardCommitMaxTimeMixedAdds
   [junit4]   2> 25081 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 25160 WARN  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 25162 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 25184 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 25199 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 25600 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 25625 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.IndexSchema [collection1] unique key field: id
   [junit4]   2> 25734 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1
   [junit4]   2> 25735 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.update.SoftAutoCommitTest_BF4DC0948A35D7ED-001/init-core-data-001/]
   [junit4]   2> 25752 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrCore New index directory detected: old=null new=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.update.SoftAutoCommitTest_BF4DC0948A35D7ED-001/init-core-data-001/index/
   [junit4]   2> 25760 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 1 plugins of type queryResponseWriter: [xml]
   [junit4]   2> 25762 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 1 plugins of type queryParser: [foo]
   [junit4]   2> 25763 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 4 plugins of type searchComponent: [spellcheck, termsComp, tvComponent, highlight]
   [junit4]   2> 25764 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 25764 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 25764 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 25764 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 25764 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 25765 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 25765 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 25766 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 25766 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 25766 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 25766 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 25803 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 39 plugins of type requestHandler: [/update, /update/json, /update/csv, /update/json/docs, /config, /schema, /replication, /get, /admin/ping, /admin/segments, /admin/luke, /admin/system, /admin/mbeans, /admin/plugins, /admin/threads, /admin/properties, /admin/logging, /admin/file, /export, /graph, /stream, /sql, /terms, standard, dismax, mock, defaults, lazy, spellCheckCompRH, spellCheckCompRH_Direct, spellCheckWithWordbreak, spellCheckWithWordbreak_Direct, spellCheckCompRH1, mltrh, tvrh, /mlt, /debug/dump, /search-facet-def, /search-facet-invariants]
   [junit4]   2> 25803 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,standard,spellCheckCompRH,lazy,/update/json/docs,/admin/luke,/export,spellCheckWithWordbreak_Direct,tvrh,/get,/admin/properties,/update/json,mltrh,/admin/threads,/search-facet-def,/update/csv,/sql,/graph,/search-facet-invariants,/admin/segments,spellCheckCompRH1,/mlt,/admin/system,/replication,dismax,defaults,/config,/stream,/schema,/admin/plugins,/admin/logging,spellCheckWithWordbreak,mock,/admin/ping,/update,/admin/file,/terms,spellCheckCompRH_Direct,/debug/dump
   [junit4]   2> 25910 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 25910 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 25910 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3419a255[collection1] main]
   [junit4]   2> 25911 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.r.RestManager Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 25911 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 25911 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 25911 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 25928 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 25957 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 25958 INFO  (searcherExecutor-28-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 25959 INFO  (searcherExecutor-28-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 25959 INFO  (searcherExecutor-28-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 25959 INFO  (searcherExecutor-28-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 25959 INFO  (searcherExecutor-28-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 25960 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.UpdateLog Looking up max value of version field to seed version buckets
   [junit4]   2> 25964 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.VersionInfo Refreshing highest value of _version_ for 65536 version buckets from index
   [junit4]   2> 25964 INFO  (searcherExecutor-28-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 25964 INFO  (searcherExecutor-28-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 25965 INFO  (searcherExecutor-28-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 25965 INFO  (searcherExecutor-28-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 25965 INFO  (searcherExecutor-28-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3419a255[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(7.0.0):c1)))}
   [junit4]   2> 25979 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.VersionInfo Found MAX value 1546413752734187520 from Terms for _version_ in index
   [junit4]   2> 25983 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.UpdateLog Took 23.0ms to seed version buckets with highest version 1546413752734187520
   [junit4]   2> 25998 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.DefaultSolrCoreState Closing old IndexWriter... core=collection1
   [junit4]   2> 26000 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=20, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4716157541174105]
   [junit4]   2> 26007 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@1ccd07a0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@417b113),segFN=segments_3,generation=3}
   [junit4]   2> 26007 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 3
   [junit4]   2> 26007 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 26012 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@23be85e3[collection1] main]
   [junit4]   2> 26013 INFO  (searcherExecutor-28-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@23be85e3[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(7.0.0):c1)))}
   [junit4]   2> 26013 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@e0c4225
   [junit4]   2> 26017 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.DirectUpdateHandler2 closing DirectUpdateHandler2{commits=4,autocommit maxTime=1200ms,autocommits=1,soft autocommit maxTime=500ms,soft autocommits=2,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=2,cumulative_deletesById=1,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=294,transaction_logs_total_number=2}
   [junit4]   2> 26024 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrCore [collection1] Closing main searcher on request.
   [junit4]   2> 26044 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[529 (1546413756604481536)]} 0 18
   [junit4]   2> 26732 INFO  (commitScheduler-31-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 26761 INFO  (commitScheduler-31-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@69703024[collection1] main]
   [junit4]   2> 26762 INFO  (searcherExecutor-28-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@69703024[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(7.0.0):c1) Uninverting(_2(7.0.0):c1)))}
   [junit4]   2> 26763 INFO  (commitScheduler-31-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 26774 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[530 (1546413757386719232)]} 0 2
   [junit4]   2> 27256 INFO  (commitScheduler-30-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 27256 INFO  (commitScheduler-30-thread-1) [    x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@334ba346
   [junit4]   2> 27294 INFO  (commitScheduler-30-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@1ccd07a0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@417b113),segFN=segments_3,generation=3}
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@1ccd07a0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@417b113),segFN=segments_4,generation=4}
   [junit4]   2> 27294 INFO  (commitScheduler-30-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 4
   [junit4]   2> 27297 INFO  (commitScheduler-30-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@601550bb[collection1] main]
   [junit4]   2> 27297 INFO  (commitScheduler-30-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 27300 INFO  (searcherExecutor-28-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@601550bb[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(7.0.0):c1) Uninverting(_2(7.0.0):c1) Uninverting(_3(7.0.0):c1)))}
   [junit4]   2> 30566 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.SolrTestCaseJ4 ###Ending testSoftAndHardCommitMaxTimeMixedAdds
   [junit4]   2> 30569 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.SolrTestCaseJ4 ###Starting testSoftAndHardCommitMaxTimeRapidAdds
   [junit4]   2> 30570 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 30978 WARN  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 30985 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 31209 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 31308 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 32195 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 32228 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.IndexSchema [collection1] unique key field: id
   [junit4]   2> 32379 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1
   [junit4]   2> 32408 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.update.SoftAutoCommitTest_BF4DC0948A35D7ED-001/init-core-data-001/]
   [junit4]   2> 32409 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrCore New index directory detected: old=null new=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.update.SoftAutoCommitTest_BF4DC0948A35D7ED-001/init-core-data-001/index/
   [junit4]   2> 32464 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 1 plugins of type queryResponseWriter: [xml]
   [junit4]   2> 32466 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 1 plugins of type queryParser: [foo]
   [junit4]   2> 32467 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 4 plugins of type searchComponent: [spellcheck, termsComp, tvComponent, highlight]
   [junit4]   2> 32467 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 32467 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 32467 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 32467 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 32467 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 32467 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 32467 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 32468 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 32468 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 32478 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 32479 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 32676 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 39 plugins of type requestHandler: [/update, /update/json, /update/csv, /update/json/docs, /config, /schema, /replication, /get, /admin/ping, /admin/segments, /admin/luke, /admin/system, /admin/mbeans, /admin/plugins, /admin/threads, /admin/properties, /admin/logging, /admin/file, /export, /graph, /stream, /sql, /terms, standard, dismax, mock, defaults, lazy, spellCheckCompRH, spellCheckCompRH_Direct, spellCheckWithWordbreak, spellCheckWithWordbreak_Direct, spellCheckCompRH1, mltrh, tvrh, /mlt, /debug/dump, /search-facet-def, /search-facet-invariants]
   [junit4]   2> 32677 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,standard,spellCheckCompRH,lazy,/update/json/docs,/admin/luke,/export,spellCheckWithWordbreak_Direct,tvrh,/get,/admin/properties,/update/json,mltrh,/admin/threads,/search-facet-def,/update/csv,/sql,/graph,/search-facet-invariants,/admin/segments,spellCheckCompRH1,/mlt,/admin/system,/replication,dismax,defaults,/config,/stream,/schema,/admin/plugins,/admin/logging,spellCheckWithWordbreak,mock,/admin/ping,/update,/admin/file,/terms,spellCheckCompRH_Direct,/debug/dump
   [junit4]   2> 32680 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 32680 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 32693 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2d6d6afc[collection1] main]
   [junit4]   2> 32694 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.r.RestManager Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 32694 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 32694 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 32694 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 32698 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 32734 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 32734 INFO  (searcherExecutor-32-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 32736 INFO  (searcherExecutor-32-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 32736 INFO  (searcherExecutor-32-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [ju

[...truncated too long message...]

til.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2>    2) Thread[id=256394, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrVersionReplicationTest]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Sep 25, 2016 5:27:13 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=256390, name=searcherExecutor-5664-thread-1, state=WAITING, group=TGRP-CdcrVersionReplicationTest]
   [junit4]   2>    2) Thread[id=256394, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrVersionReplicationTest]
   [junit4]   2> Sep 25, 2016 5:27:16 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=256390, name=searcherExecutor-5664-thread-1, state=WAITING, group=TGRP-CdcrVersionReplicationTest]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {}, docValues:{}, maxPointsInLeafNode=780, maxMBSortInHeap=5.38963644527261, sim=ClassicSimilarity, locale=es-GT, timezone=Atlantic/Cape_Verde
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=2,free=71710832,total=498073600
   [junit4]   2> NOTE: All tests run in this JVM: [DisMaxRequestHandlerTest, SharedFSAutoReplicaFailoverTest, SolrRequestParserTest, TestQuerySenderNoQuery, TestSolrDeletionPolicy2, TestJmxIntegration, TestSubQueryTransformer, DistributedQueryElevationComponentTest, TestCloudSchemaless, TestSolrQueryResponse, TestFieldSortValues, TestRandomDVFaceting, TestFastWriter, TestJoin, ActionThrottleTest, TestExactStatsCache, ClusterStateTest, TestSolrCoreProperties, TestAtomicUpdateErrorCases, AnalysisErrorHandlingTest, TestExclusionRuleCollectionAccess, TestReloadDeadlock, TestSolr4Spatial, SolrCLIZkUtilsTest, TestOverriddenPrefixQueryForCustomFieldType, CdcrBootstrapTest, SuggestComponentTest, DistribCursorPagingTest, CollectionsAPISolrJTest, DistributedQueueTest, TestConfigSetImmutable, TestFieldTypeResource, ZkStateReaderTest, TestLeaderInitiatedRecoveryThread, TestFaceting, MBeansHandlerTest, TestUseDocValuesAsStored2, TestPerFieldSimilarity, TestSQLHandler, TestBadConfig, DocumentAnalysisRequestHandlerTest, CleanupOldIndexTest, TestNumericTerms64, VersionInfoTest, TestMaxScoreQueryParser, TestNonDefinedSimilarityFactory, DeleteReplicaTest, StatsReloadRaceTest, TestBulkSchemaConcurrent, BasicDistributedZkTest, TestDistributedSearch, ClassificationUpdateProcessorFactoryTest, DistributedSpellCheckComponentTest, TestFieldCollectionResource, ResourceLoaderTest, SolrCoreTest, HighlighterConfigTest, HttpPartitionTest, TestCodecSupport, BasicZkTest, FastVectorHighlighterTest, TestComponentsName, LeaderInitiatedRecoveryOnCommitTest, TestFieldCacheWithThreads, RecoveryAfterSoftCommitTest, TestConfigSetsAPI, TestHashQParserPlugin, SSLMigrationTest, DateFieldTest, TestManagedStopFilterFactory, IgnoreCommitOptimizeUpdateProcessorFactoryTest, UUIDFieldTest, TestNamedUpdateProcessors, TestPivotHelperCode, BooleanFieldTest, TestCloudPivotFacet, TestCrossCoreJoin, TestRemoteStreaming, PolyFieldTest, OverseerCollectionConfigSetProcessorTest, TestCryptoKeys, CheckHdfsIndexTest, RegexBoostProcessorTest, LeaderElectionIntegrationTest, TestDocumentBuilder, DistributedQueryComponentCustomSortTest, TestFieldCacheSort, TestDefaultStatsCache, TestGraphMLResponseWriter, CursorMarkTest, TestTolerantUpdateProcessorCloud, CollectionStateFormat2Test, TestReplicationHandlerBackup, TestFieldCache, OverseerStatusTest, DistributedFacetPivotSmallAdvancedTest, TestBinaryResponseWriter, QueryEqualityTest, StandardRequestHandlerTest, TestSolrQueryParserDefaultOperatorResource, BigEndianAscendingWordSerializerTest, TestLazyCores, CursorPagingTest, CdcrVersionReplicationTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=CdcrVersionReplicationTest -Dtests.seed=BF4DC0948A35D7ED -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=es-GT -Dtests.timezone=Atlantic/Cape_Verde -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J0 | CdcrVersionReplicationTest (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.CdcrVersionReplicationTest: 
   [junit4]    >    1) Thread[id=256390, name=searcherExecutor-5664-thread-1, state=WAITING, group=TGRP-CdcrVersionReplicationTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    2) Thread[id=256394, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrVersionReplicationTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BF4DC0948A35D7ED]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=256390, name=searcherExecutor-5664-thread-1, state=WAITING, group=TGRP-CdcrVersionReplicationTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BF4DC0948A35D7ED]:0)
   [junit4] Completed [467/636 (7!)] on J0 in 787.70s, 1 test, 3 errors <<< FAILURES!

[...truncated 509 lines...]
   [junit4] JVM J1: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20160925_034407_301.sysout
   [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid25432.hprof ...
   [junit4] Heap dump file created [511477987 bytes in 10.210 secs]
   [junit4] <<< JVM J1: EOF ----

[...truncated 3535 lines...]
ERROR: Step ‘Archive the artifacts’ failed: no workspace for Lucene-Solr-NightlyTests-master #1121
ERROR: Step ‘Publish JUnit test result report’ failed: no workspace for Lucene-Solr-NightlyTests-master #1121
ERROR: lucene is offline; cannot locate JDK 1.8 (latest)
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
ERROR: lucene is offline; cannot locate JDK 1.8 (latest)
ERROR: lucene is offline; cannot locate JDK 1.8 (latest)




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1120 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1120/

270 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.BasicFunctionalityTest

Error Message:
java.lang.NullPointerException

Stack Trace:
java.lang.RuntimeException: java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([219E416F25A4A93]:0)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:75)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:602)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:595)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:437)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:426)
	at org.apache.solr.BasicFunctionalityTest.beforeTests(BasicFunctionalityTest.java:66)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:811)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:178)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:142)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:171)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:73)
	... 29 more


FAILED:  junit.framework.TestSuite.org.apache.solr.ConvertedLegacyTest

Error Message:
java.lang.NullPointerException

Stack Trace:
java.lang.RuntimeException: java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([219E416F25A4A93]:0)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:75)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:602)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:595)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:437)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:426)
	at org.apache.solr.ConvertedLegacyTest.beforeTests(ConvertedLegacyTest.java:37)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:811)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:178)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:142)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:171)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:73)
	... 29 more


FAILED:  org.apache.solr.DistributedIntervalFacetingTest.test

Error Message:
Error from server at http://127.0.0.1:58331//collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.NullPointerException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: null,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: null  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1119)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:252)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:414)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:266)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:217)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(Thread.java:745) Caused by: java.lang.NullPointerException  at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)  at org.apache.solr.core.SolrResourceLoader.&lt;init&gt;(SolrResourceLoader.java:178)  at org.apache.solr.core.ConfigSetService$Default.createCoreResourceLoader(ConfigSetService.java:159)  at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:74)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:862)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:520)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:58331//collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.NullPointerException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: null,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: null
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1119)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:252)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:414)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:266)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:217)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)
	at org.apache.solr.core.SolrResourceLoader.&lt;init&gt;(SolrResourceLoader.java:178)
	at org.apache.solr.core.ConfigSetService$Default.createCoreResourceLoader(ConfigSetService.java:159)
	at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:74)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:862)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:520)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([219E416F25A4A93:8A4DDBCC5CA6276B]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.BaseDistributedSearchTestCase.del(BaseDistributedSearchTestCase.java:540)
	at org.apache.solr.DistributedIntervalFacetingTest.test(DistributedIntervalFacetingTest.java:44)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsRepeatStatement.callStatement(BaseDistributedSearchTestCase.java:1011)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.EchoParamsTest

Error Message:
java.lang.NullPointerException

Stack Trace:
java.lang.RuntimeException: java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([219E416F25A4A93]:0)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:75)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:602)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:595)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:437)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:426)
	at org.apache.solr.EchoParamsTest.beforeClass(EchoParamsTest.java:29)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:811)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:178)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:142)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:171)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:73)
	... 29 more


FAILED:  junit.framework.TestSuite.org.apache.solr.MinimalSchemaTest

Error Message:
java.lang.NullPointerException

Stack Trace:
java.lang.RuntimeException: java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([219E416F25A4A93]:0)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:75)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:602)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:595)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:437)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:426)
	at org.apache.solr.MinimalSchemaTest.beforeClass(MinimalSchemaTest.java:39)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:811)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:178)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:142)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:171)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:73)
	... 29 more


FAILED:  junit.framework.TestSuite.org.apache.solr.SampleTest

Error Message:
java.lang.NullPointerException

Stack Trace:
java.lang.RuntimeException: java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([219E416F25A4A93]:0)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:75)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:602)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:595)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:437)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:426)
	at org.apache.solr.SampleTest.beforeClass(SampleTest.java:41)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:811)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:178)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:142)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:171)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:73)
	... 29 more


FAILED:  junit.framework.TestSuite.org.apache.solr.SolrInfoMBeanTest

Error Message:
java.lang.NullPointerException

Stack Trace:
java.lang.RuntimeException: java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([219E416F25A4A93]:0)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:75)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:602)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:595)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:437)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:426)
	at org.apache.solr.SolrInfoMBeanTest.beforeClass(SolrInfoMBeanTest.java:40)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:811)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:178)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:142)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:171)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:73)
	... 29 more


FAILED:  junit.framework.TestSuite.org.apache.solr.SolrTestCaseJ4Test

Error Message:
Can't load schema /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.SolrTestCaseJ4Test_219E416F25A4A93-001/tempDir-001/core1/conf/schema-tiny.xml: null

Stack Trace:
org.apache.solr.common.SolrException: Can't load schema /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.SolrTestCaseJ4Test_219E416F25A4A93-001/tempDir-001/core1/conf/schema-tiny.xml: null
	at __randomizedtesting.SeedInfo.seed([219E416F25A4A93]:0)
	at org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:607)
	at org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:183)
	at org.apache.solr.schema.IndexSchemaFactory.create(IndexSchemaFactory.java:56)
	at org.apache.solr.schema.IndexSchemaFactory.buildIndexSchema(IndexSchemaFactory.java:75)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:96)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:605)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:595)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:437)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:444)
	at org.apache.solr.SolrTestCaseJ4Test.beforeClass(SolrTestCaseJ4Test.java:52)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:811)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:583)
	... 33 more


FAILED:  org.apache.solr.TestCursorMarkWithoutUniqueKey.test

Error Message:
java.lang.NullPointerException

Stack Trace:
java.lang.RuntimeException: java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([219E416F25A4A93:8A4DDBCC5CA6276B]:0)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:75)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:602)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:595)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:437)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:426)
	at org.apache.solr.TestCursorMarkWithoutUniqueKey.beforeSetupCore(TestCursorMarkWithoutUniqueKey.java:38)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:905)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:178)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:142)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:171)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:73)
	... 43 more


FAILED:  org.apache.solr.TestDistributedGrouping.test

Error Message:
Error from server at https://127.0.0.1:55533//collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.NullPointerException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: null,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: null  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1119)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:252)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:414)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:266)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:217)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(Thread.java:745) Caused by: java.lang.NullPointerException  at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)  at org.apache.solr.core.SolrResourceLoader.&lt;init&gt;(SolrResourceLoader.java:178)  at org.apache.solr.core.ConfigSetService$Default.createCoreResourceLoader(ConfigSetService.java:159)  at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:74)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:862)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:520)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:55533//collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.NullPointerException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: null,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: null
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1119)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:252)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:414)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:266)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:217)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)
	at org.apache.solr.core.SolrResourceLoader.&lt;init&gt;(SolrResourceLoader.java:178)
	at org.apache.solr.core.ConfigSetService$Default.createCoreResourceLoader(ConfigSetService.java:159)
	at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:74)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:862)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:520)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([219E416F25A4A93:8A4DDBCC5CA6276B]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.BaseDistributedSearchTestCase.del(BaseDistributedSearchTestCase.java:540)
	at org.apache.solr.TestDistributedGrouping.test(TestDistributedGrouping.java:53)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsRepeatStatement.callStatement(BaseDistributedSearchTestCase.java:1011)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.TestDistributedMissingSort.test

Error Message:
Error from server at https://127.0.0.1:46248//collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.NullPointerException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: null,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: null  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1119)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:252)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:414)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:266)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:217)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(Thread.java:745) Caused by: java.lang.NullPointerException  at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)  at org.apache.solr.core.SolrResourceLoader.&lt;init&gt;(SolrResourceLoader.java:178)  at org.apache.solr.core.ConfigSetService$Default.createCoreResourceLoader(ConfigSetService.java:159)  at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:74)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:862)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:520)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:46248//collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.NullPointerException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: null,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: null
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1119)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:252)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:414)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:266)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:217)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)
	at org.apache.solr.core.SolrResourceLoader.&lt;init&gt;(SolrResourceLoader.java:178)
	at org.apache.solr.core.ConfigSetService$Default.createCoreResourceLoader(ConfigSetService.java:159)
	at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:74)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:862)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:520)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([219E416F25A4A93:8A4DDBCC5CA6276B]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.BaseDistributedSearchTestCase.del(BaseDistributedSearchTestCase.java:540)
	at org.apache.solr.TestDistributedMissingSort.index(TestDistributedMissingSort.java:48)
	at org.apache.solr.TestDistributedMissingSort.test(TestDistributedMissingSort.java:42)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsRepeatStatement.callStatement(BaseDistributedSearchTestCase.java:1011)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.TestDistributedSearch.test

Error Message:
Error from server at http://127.0.0.1:49317//collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.NullPointerException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: null,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: null  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1119)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:252)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:414)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:266)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:217)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(Thread.java:745) Caused by: java.lang.NullPointerException  at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)  at org.apache.solr.core.SolrResourceLoader.&lt;init&gt;(SolrResourceLoader.java:178)  at org.apache.solr.core.ConfigSetService$Default.createCoreResourceLoader(ConfigSetService.java:159)  at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:74)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:862)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:520)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:49317//collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.NullPointerException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: null,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: null
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1119)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:252)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:414)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:266)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:217)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)
	at org.apache.solr.core.SolrResourceLoader.&lt;init&gt;(SolrResourceLoader.java:178)
	at org.apache.solr.core.ConfigSetService$Default.createCoreResourceLoader(ConfigSetService.java:159)
	at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:74)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:862)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:520)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([219E416F25A4A93:8A4DDBCC5CA6276B]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.BaseDistributedSearchTestCase.del(BaseDistributedSearchTestCase.java:540)
	at org.apache.solr.TestDistributedSearch.test(TestDistributedSearch.java:109)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsRepeatStatement.callStatement(BaseDistributedSearchTestCase.java:1011)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.TestGroupingSearch

Error Message:
java.lang.NullPointerException

Stack Trace:
java.lang.RuntimeException: java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([219E416F25A4A93]:0)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:75)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:602)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:595)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:437)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:426)
	at org.apache.solr.TestGroupingSearch.beforeTests(TestGroupingSearch.java:70)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:811)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:178)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:142)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:171)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:73)
	... 29 more


FAILED:  org.apache.solr.TestSimpleTrackingShardHandler.testSolrXmlOverrideAndCorrectShardHandler

Error Message:
Error from server at https://127.0.0.1:54893/_cso/x/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /_cso/x/collection1/select. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.NullPointerException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: null,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: null  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1119)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:252)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:414)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:266)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:217)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(Thread.java:745) Caused by: java.lang.NullPointerException  at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)  at org.apache.solr.core.SolrResourceLoader.&lt;init&gt;(SolrResourceLoader.java:178)  at org.apache.solr.core.ConfigSetService$Default.createCoreResourceLoader(ConfigSetService.java:159)  at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:74)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:862)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:520)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:54893/_cso/x/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>

[...truncated too long message...]

b/apacheds-interceptors-event-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/bcprov-jdk15-1.45.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/cglib-nodep-2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/commons-math3-3.4.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/easymock-3.0.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-common-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-hdfs-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-minikdc-2.7.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-annotations-2.5.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-core-asl-1.9.13.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-databind-2.5.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-mapper-asl-1.9.13.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/mockito-core-1.9.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/objenesis-1.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/presto-parser-0.122.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/slice-0.10.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/analysis/icu/lucene-analyzers-icu-7.0.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/analysis-extras/lib/icu4j-56.1.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_102/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.3.4.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20160922_223528_905.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20160922_223528_905.suites -stdin
   [junit4] ERROR: JVM J2 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1501)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:121)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:955)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:952)
   [junit4] 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4] 	at java.lang.Thread.run(Thread.java:745)

BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:707: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build.xml:252: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/common-build.xml:536: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:1443: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:984: At least one slave process threw an exception, first: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.

Total time: 513 minutes 47 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1119 - Failure

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1119/

11 tests failed.
FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=55549, name=collection2, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=55549, name=collection2, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:59186/vlp
	at __randomizedtesting.SeedInfo.seed([8EB1B6904EB41305]:0)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:998)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:59186/vlp
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:415)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:367)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1289)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1059)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1001)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1599)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1620)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:988)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 11 more


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
Timeout waiting for all live and active

Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
	at __randomizedtesting.SeedInfo.seed([8EB1B6904EB41305:6E5894AE0487EFD]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:309)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:127)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.core.TestDynamicLoading.testDynamicLoading

Error Message:
Could not get expected value  'X val changed' for path 'x' full output: {   "responseHeader":{     "status":0,     "QTime":0},   "params":{"wt":"json"},   "context":{     "webapp":"",     "path":"/test1",     "httpMethod":"GET"},   "class":"org.apache.solr.core.BlobStoreTestRequestHandler",   "x":null},  from server:  null

Stack Trace:
java.lang.AssertionError: Could not get expected value  'X val changed' for path 'x' full output: {
  "responseHeader":{
    "status":0,
    "QTime":0},
  "params":{"wt":"json"},
  "context":{
    "webapp":"",
    "path":"/test1",
    "httpMethod":"GET"},
  "class":"org.apache.solr.core.BlobStoreTestRequestHandler",
  "x":null},  from server:  null
	at __randomizedtesting.SeedInfo.seed([8EB1B6904EB41305:56FC9BC7B969B6A5]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.core.TestSolrConfigHandler.testForResponseElement(TestSolrConfigHandler.java:535)
	at org.apache.solr.core.TestDynamicLoading.testDynamicLoading(TestDynamicLoading.java:249)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1474364963730,generation=2,filelist=[_5.fdt, _5.fdx, _5.fnm, _5.nvd, _5.nvm, _5.si, _5_Lucene50_0.doc, _5_Lucene50_0.tim, _5_Lucene50_0.tip, _7.fdt, _7.fdx, _7.fnm, _7.nvd, _7.nvm, _7.si, _7_Lucene50_0.doc, _7_Lucene50_0.tim, _7_Lucene50_0.tip, _8.cfe, _8.cfs, _8.si, segments_2]}]> but was:<[{indexVersion=1474364963730,generation=2,filelist=[_5.fdt, _5.fdx, _5.fnm, _5.nvd, _5.nvm, _5.si, _5_Lucene50_0.doc, _5_Lucene50_0.tim, _5_Lucene50_0.tip, _7.fdt, _7.fdx, _7.fnm, _7.nvd, _7.nvm, _7.si, _7_Lucene50_0.doc, _7_Lucene50_0.tim, _7_Lucene50_0.tip, _8.cfe, _8.cfs, _8.si, segments_2]}, {indexVersion=1474364963730,generation=3,filelist=[_a.fdt, _a.fdx, _a.fnm, _a.nvd, _a.nvm, _a.si, _a_Lucene50_0.doc, _a_Lucene50_0.tim, _a_Lucene50_0.tip, segments_3]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1474364963730,generation=2,filelist=[_5.fdt, _5.fdx, _5.fnm, _5.nvd, _5.nvm, _5.si, _5_Lucene50_0.doc, _5_Lucene50_0.tim, _5_Lucene50_0.tip, _7.fdt, _7.fdx, _7.fnm, _7.nvd, _7.nvm, _7.si, _7_Lucene50_0.doc, _7_Lucene50_0.tim, _7_Lucene50_0.tip, _8.cfe, _8.cfs, _8.si, segments_2]}]> but was:<[{indexVersion=1474364963730,generation=2,filelist=[_5.fdt, _5.fdx, _5.fnm, _5.nvd, _5.nvm, _5.si, _5_Lucene50_0.doc, _5_Lucene50_0.tim, _5_Lucene50_0.tip, _7.fdt, _7.fdx, _7.fnm, _7.nvd, _7.nvm, _7.si, _7_Lucene50_0.doc, _7_Lucene50_0.tim, _7_Lucene50_0.tip, _8.cfe, _8.cfs, _8.si, segments_2]}, {indexVersion=1474364963730,generation=3,filelist=[_a.fdt, _a.fdx, _a.fnm, _a.nvd, _a.nvm, _a.si, _a_Lucene50_0.doc, _a_Lucene50_0.tim, _a_Lucene50_0.tip, segments_3]}]>
	at __randomizedtesting.SeedInfo.seed([8EB1B6904EB41305:AB66ADA03EFC1D06]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1227)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.handler.component.DistributedFacetPivotLongTailTest.test

Error Message:
Error from server at https://127.0.0.1:49359//collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    java.lang.OutOfMemoryError: GC overhead limit exceeded</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:49359//collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    java.lang.OutOfMemoryError: GC overhead limit exceeded</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([8EB1B6904EB41305:6E5894AE0487EFD]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:152)
	at org.apache.solr.handler.component.DistributedFacetPivotLongTailTest.test(DistributedFacetPivotLongTailTest.java:71)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.lucene.search.TestFuzzyQuery.testRandom

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([75FB2BA2C4CAA695]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.search.TestFuzzyQuery

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([75FB2BA2C4CAA695]:0)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart

Error Message:
Timeout while trying to assert number of documents @ source_collection

Stack Trace:
java.lang.AssertionError: Timeout while trying to assert number of documents @ source_collection
	at __randomizedtesting.SeedInfo.seed([8EB1B6904EB41305:D2AC51118094A733]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:273)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart(CdcrReplicationDistributedZkTest.java:257)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.AssertionError: expected:<110> but was:<106>
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:264)
	... 42 more


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterLeaderChange

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard1

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard1
	at __randomizedtesting.SeedInfo.seed([8EB1B6904EB41305:5C41FA73101BB537]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterLeaderChange(CdcrReplicationDistributedZkTest.java:305)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
	at __randomizedtesting.SeedInfo.seed([8EB1B6904EB41305]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.GeneratedMethodAccessor84.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest:     1) Thread[id=119548, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=119553, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=119543, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=119573, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=119558, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=119538, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=119563, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=119578, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=119568, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=119533, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 
   1) Thread[id=119548, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=119553, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=119543, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=119573, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=119558, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=119538, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=119563, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=119578, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=119568, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=119533, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([8EB1B6904EB41305]:0)




Build Log:
[...truncated 1631 lines...]
   [junit4] Suite: org.apache.lucene.search.TestFuzzyQuery
   [junit4]   2> Rgs 20, 2016 3:56:40 AM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.search.TestFuzzyQuery
   [junit4]   2>    1) Thread[id=47, name=SUITE-TestFuzzyQuery-seed#[75FB2BA2C4CAA695], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1556)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:688)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:685)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:685)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:701)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:681)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:64)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:414)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:681)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:140)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:591)
   [junit4]   2>    2) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1249)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1323)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:601)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:450)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:243)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:354)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:10)
   [junit4]   2>    3) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:47)
   [junit4]   2>    4) Thread[id=48, name=TEST-TestFuzzyQuery.testRandom-seed#[75FB2BA2C4CAA695], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2>         at java.lang.Object.hashCode(Native Method)
   [junit4]   2>         at java.util.WeakHashMap.hash(WeakHashMap.java:298)
   [junit4]   2>         at java.util.WeakHashMap.getEntry(WeakHashMap.java:426)
   [junit4]   2>         at java.util.WeakHashMap.containsKey(WeakHashMap.java:417)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:254)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:151)
   [junit4]   2>         at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:729)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.randomSimpleString(TestFuzzyQuery.java:506)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.testRandom(TestFuzzyQuery.java:517)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestFuzzyQuery -Dtests.method=testRandom -Dtests.seed=75FB2BA2C4CAA695 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=lt-LT -Dtests.timezone=America/Argentina/Rio_Gallegos -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   7199s J1 | TestFuzzyQuery.testRandom <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([75FB2BA2C4CAA695]:0)
   [junit4]   2> Rgs 20, 2016 3:56:40 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> Rgs 20, 2016 3:57:00 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.search.TestFuzzyQuery: 
   [junit4]   2>    1) Thread[id=48, name=TEST-TestFuzzyQuery.testRandom-seed#[75FB2BA2C4CAA695], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2>         at java.util.WeakHashMap.getTable(WeakHashMap.java:350)
   [junit4]   2>         at java.util.WeakHashMap.get(WeakHashMap.java:397)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.getPerThread(RandomizedContext.java:315)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.getRandomness(RandomizedContext.java:110)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.getRandom(RandomizedContext.java:132)
   [junit4]   2>         at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:729)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.randomSimpleString(TestFuzzyQuery.java:506)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.testRandom(TestFuzzyQuery.java:517)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Rgs 20, 2016 3:57:00 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=48, name=TEST-TestFuzzyQuery.testRandom-seed#[75FB2BA2C4CAA695], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2> Rgs 20, 2016 3:57:03 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=48, name=TEST-TestFuzzyQuery.testRandom-seed#[75FB2BA2C4CAA695], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2>         at java.lang.Object.hashCode(Native Method)
   [junit4]   2>         at java.util.WeakHashMap.hash(WeakHashMap.java:298)
   [junit4]   2>         at java.util.WeakHashMap.get(WeakHashMap.java:396)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.getPerThread(RandomizedContext.java:315)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.getRandomness(RandomizedContext.java:110)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.getRandom(RandomizedContext.java:132)
   [junit4]   2>         at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:729)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.randomSimpleString(TestFuzzyQuery.java:506)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.testRandom(TestFuzzyQuery.java:517)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {field=FSTOrd50}, docValues:{}, maxPointsInLeafNode=365, maxMBSortInHeap=5.939115795628311, sim=ClassicSimilarity, locale=lt-LT, timezone=America/Argentina/Rio_Gallegos
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=2,free=204973928,total=244318208
   [junit4]   2> NOTE: All tests run in this JVM: [TestRecyclingIntBlockAllocator, TestSumDocFreq, TestFuzzyQuery]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestFuzzyQuery -Dtests.seed=75FB2BA2C4CAA695 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=lt-LT -Dtests.timezone=America/Argentina/Rio_Gallegos -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J1 | TestFuzzyQuery (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([75FB2BA2C4CAA695]:0)
   [junit4] Completed [364/433 (1!)] on J1 in 7223.48s, 3 tests, 2 errors <<< FAILURES!

[...truncated 10031 lines...]
   [junit4] Suite: org.apache.solr.handler.TestReplicationHandler
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/init-core-data-001
   [junit4]   2> 1484458 INFO  (SUITE-TestReplicationHandler-seed#[8EB1B6904EB41305]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
   [junit4]   2> 1484459 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testEmptyCommits
   [junit4]   2> 1484460 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/collection1
   [junit4]   2> 1484475 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 1484554 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@a4a36aa{/solr,null,AVAILABLE}
   [junit4]   2> 1484555 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.e.j.s.ServerConnector Started ServerConnector@7e702bb7{HTTP/1.1,[http/1.1]}{127.0.0.1:40836}
   [junit4]   2> 1484555 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.e.j.s.Server Started @1492810ms
   [junit4]   2> 1484555 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/collection1/data, hostContext=/solr, hostPort=40836}
   [junit4]   2> 1484555 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 1484555 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001'
   [junit4]   2> 1484555 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1484555 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 1484556 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/solr.xml
   [junit4]   2> 1484560 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/.
   [junit4]   2> 1484561 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.CoreContainer New CoreContainer 1219314704
   [junit4]   2> 1484561 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001]
   [junit4]   2> 1484561 WARN  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.CoreContainer Couldn't add files from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/lib to classpath: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/lib
   [junit4]   2> 1484561 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,
   [junit4]   2> 1484567 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.u.UpdateShardHandler Created UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&maxConnections=100000&maxConnectionsPerHost=100000
   [junit4]   2> 1484567 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1484567 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1484567 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 1484567 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 1484568 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/.
   [junit4]   2> 1484569 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection1, config=solrconfig.xml, loadOnStartup=true, schema=schema.xml, configSetProperties=configsetprops.json, transient=false, dataDir=data/}
   [junit4]   2> 1484569 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.CorePropertiesLocator Found core collection1 in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/./collection1
   [junit4]   2> 1484569 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions
   [junit4]   2> 1484569 INFO  (coreLoadExecutor-2198-thread-1) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/collection1'
   [junit4]   2> 1484586 INFO  (coreLoadExecutor-2198-thread-1) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1484586 INFO  (coreLoadExecutor-2198-thread-1) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 1484590 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.s.SolrDispatchFilter user.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0
   [junit4]   2> 1484590 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 1484599 INFO  (coreLoadExecutor-2198-thread-1) [    ] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 1484926 WARN  (coreLoadExecutor-2198-thread-1) [    ] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1484939 INFO  (coreLoadExecutor-2198-thread-1) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1484945 INFO  (coreLoadExecutor-2198-thread-1) [    ] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1484956 INFO  (coreLoadExecutor-2198-thread-1) [    ] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1484967 INFO  (coreLoadExecutor-2198-thread-1) [    ] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 1484969 INFO  (coreLoadExecutor-2198-thread-1) [    ] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properties: Can't find resource 'configsetprops.json' in classpath or '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/collection1'
   [junit4]   2> 1484969 INFO  (coreLoadExecutor-2198-thread-1) [    ] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/./collection1
   [junit4]   2> 1484969 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1484969 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/./collection1/data/]
   [junit4]   2> 1484969 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.SolrCore JMX monitoring not detected for core: collection1
   [junit4]   2> 1484969 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/./collection1/data/snapshot_metadata
   [junit4]   2> 1484970 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.s.SolrSnapshotMetaDataManager Loading from snapshot metadata file...
   [junit4]   2> 1484982 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/./collection1/data
   [junit4]   2> 1484986 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.SolrCore New index directory detected: old=null new=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/./collection1/data/index/
   [junit4]   2> 1484986 WARN  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1] Solr index directory '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/./collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1484986 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/./collection1/data/index
   [junit4]   2> 1484987 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=12.0078125, floorSegmentMB=0.3564453125, forceMergeDeletesPctAllowed=7.7738484720713386, segmentsPerTier=10.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1484994 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@433a7d2 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5b6d9402; maxCacheMB=0.7265625 maxMergeSizeMB=0.3115234375)),segFN=segments_1,generation=1}
   [junit4]   2> 1484994 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 1485124 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1485126 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1485127 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1485127 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1485128 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1485129 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.PluginBag requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 1485129 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,standard,lazy,/update/json/docs,/admin/luke,/export,/get,/admin/properties,/update/json,/admin/threads,/update/csv,/sql,/graph,/admin/segments,/admin/system,/replication,defaults,/config,/stream,/schema,/admin/plugins,/admin/logging,/admin/ping,/update,/admin/file,/terms
   [junit4]   2> 1485129 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.SolrCore Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1485130 INFO  (OldIndexDirectoryCleanupThreadForCore-collection1) [    x:collection1] o.a.s.c.SolrCore Looking for old index directories to cleanup for core collection1 in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/./collection1/data/
   [junit4]   2> 1485130 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1485130 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1485130 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=2, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4865483743754896]
   [junit4]   2> 1485131 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@433a7d2 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5b6d9402; maxCacheMB=0.7265625 maxMergeSizeMB=0.3115234375)),segFN=segments_1,generation=1}
   [junit4]   2> 1485131 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 1485131 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@10cf9d91[collection1] main]
   [junit4]   2> 1485131 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/collection1/conf
   [junit4]   2> 1485131 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.r.RestManager Initializing RestManager with initArgs: {storageDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/collection1/conf}
   [junit4]   2> 1485131 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using file:dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/collection1/conf
   [junit4]   2> 1485132 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 1485132 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Replication enabled for following config files: schema.xml,xslt/dummy.xsl
   [junit4]   2> 1485132 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1485132 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.CoreContainer registering core: collection1
   [junit4]   2> 1485133 INFO  (searcherExecutor-2199-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@10cf9d91[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1485664 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-002/collection1
   [junit4]   2> 1485667 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 1485671 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7cfc65c{/solr,null,AVAILABLE}
   [junit4]   2> 1485671 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.e.j.s.ServerConnector Started ServerConnector@56b80777{HTTP/1.1,[http/1.1]}{127.0.0.1:34325}
   [junit4]   2> 1485671 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.e.j.s.Server Started @1493926ms
   [junit4]   2> 1485671 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-002/collection1/data, hostContext=/solr, hostPort=34325}
   [junit4]   2> 1485672 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 1485672 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-002'
   [junit4]   2> 1485672 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1485672 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 1485678 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handle

[...truncated too long message...]

a{/,null,UNAVAILABLE}
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=DistributedFacetPivotLongTailTest -Dtests.method=test -Dtests.seed=8EB1B6904EB41305 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=ar-SA -Dtests.timezone=Etc/Universal -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR    115s J2 | DistributedFacetPivotLongTailTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:49359//collection1: Expected mime type application/octet-stream but got text/html. <html>
   [junit4]    > <head>
   [junit4]    > <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
   [junit4]    > <title>Error 500 </title>
   [junit4]    > </head>
   [junit4]    > <body>
   [junit4]    > <h2>HTTP ERROR: 500</h2>
   [junit4]    > <p>Problem accessing /collection1/update. Reason:
   [junit4]    > <pre>    java.lang.OutOfMemoryError: GC overhead limit exceeded</pre></p>
   [junit4]    > <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
   [junit4]    > </body>
   [junit4]    > </html>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([8EB1B6904EB41305:6E5894AE0487EFD]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:152)
   [junit4]    > 	at org.apache.solr.handler.component.DistributedFacetPivotLongTailTest.test(DistributedFacetPivotLongTailTest.java:71)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 7842380 INFO  (SUITE-DistributedFacetPivotLongTailTest-seed#[8EB1B6904EB41305]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.component.DistributedFacetPivotLongTailTest_8EB1B6904EB41305-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {stat_i=PostingsFormat(name=Direct), range_facet_l_dv=PostingsFormat(name=LuceneVarGapDocFreqInterval), multiDefault=BlockTreeOrds(blocksize=128), intDefault=PostingsFormat(name=Direct), foo_s=PostingsFormat(name=Direct), id=PostingsFormat(name=LuceneVarGapDocFreqInterval), range_facet_i_dv=PostingsFormat(name=Direct), range_facet_l=PostingsFormat(name=Direct), timestamp=PostingsFormat(name=Direct)}, docValues:{range_facet_l_dv=DocValuesFormat(name=Lucene54), range_facet_i_dv=DocValuesFormat(name=Memory), timestamp=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=1307, maxMBSortInHeap=6.210753653059481, sim=ClassicSimilarity, locale=ar-SA, timezone=Etc/Universal
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=133801976,total=477626368
   [junit4]   2> NOTE: All tests run in this JVM: [TestPerFieldSimilarityWithDefaultOverride, MultiThreadedOCPTest, RecoveryAfterSoftCommitTest, TestDistributedMissingSort, TestLFUCache, TestTolerantSearch, TestSolrCloudWithSecureImpersonation, HdfsThreadLeakTest, HardAutoCommitTest, HdfsCollectionsAPIDistributedZkTest, TestAuthorizationFramework, TestConfigSetImmutable, TestSolr4Spatial2, TestXIncludeConfig, ForceLeaderTest, DirectUpdateHandlerOptimizeTest, TestFilteredDocIdSet, TestJmxMonitoredMap, TestConfig, SolrCLIZkUtilsTest, TestReversedWildcardFilterFactory, TestDistribIDF, TestRequestStatusCollectionAPI, TestCoreDiscovery, TestFreeTextSuggestions, TermVectorComponentDistributedTest, TestSolrDeletionPolicy1, TestExtendedDismaxParser, TestMergePolicyConfig, AutoCommitTest, DeleteStatusTest, BitVectorTest, FullSolrCloudDistribCmdsTest, TestCollationFieldDocValues, TestDynamicFieldCollectionResource, TestAnalyzeInfixSuggestions, TestStressReorder, ResponseLogComponentTest, BaseCdcrDistributedZkTest, SpellPossibilityIteratorTest, XmlUpdateRequestHandlerTest, TestStressUserVersions, TestUtils, TestOnReconnectListenerSupport, AnalyticsMergeStrategyTest, AddSchemaFieldsUpdateProcessorFactoryTest, DistributedFacetPivotSmallTest, SSLMigrationTest, ShowFileRequestHandlerTest, ZkStateReaderTest, TestPostingsSolrHighlighter, BlockDirectoryTest, HdfsLockFactoryTest, TestSchemaManager, GraphQueryTest, HdfsBasicDistributedZkTest, UnloadDistributedZkTest, TestMiniSolrCloudCluster, TestSizeLimitedDistributedMap, TestOrdValues, TestDynamicLoading, TimeZoneUtilsTest, JSONWriterTest, TestClassicSimilarityFactory, TestCryptoKeys, UpdateParamsTest, CurrencyFieldXmlFileTest, BadIndexSchemaTest, TestSchemaNameResource, TestCollationField, StatelessScriptUpdateProcessorFactoryTest, CollectionsAPIAsyncDistributedZkTest, TestGroupingSearch, TestSolrQueryParser, TestShardHandlerFactory, TestBinaryField, SuggesterTest, TestManagedSynonymFilterFactory, TestExclusionRuleCollectionAccess, TolerantUpdateProcessorTest, TestReRankQParserPlugin, OpenExchangeRatesOrgProviderTest, PreAnalyzedFieldTest, SimpleFacetsTest, PrimUtilsTest, DistribJoinFromCollectionTest, PluginInfoTest, PingRequestHandlerTest, TestCloudPivotFacet, TestAddFieldRealTimeGet, TestBlobHandler, DistributedDebugComponentTest, SuggesterTSTTest, TestSubQueryTransformerCrossCore, TestQueryWrapperFilter, TestZkChroot, TestReload, TestSort, CdcrUpdateLogTest, FieldMutatingUpdateProcessorTest, LeaderElectionTest, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, CollectionsAPIDistributedZkTest, SyncSliceTest, LeaderElectionIntegrationTest, ClusterStateUpdateTest, ZkSolrClientTest, TestRecovery, TestHashPartitioner, DistributedTermsComponentTest, TestRangeQuery, TestSolrDeletionPolicy2, MultiTermTest, MinimalSchemaTest, TestElisionMultitermQuery, OutputWriterTest, TestFuzzyAnalyzedSuggestions, TestSolrCoreProperties, DirectSolrConnectionTest, NotRequiredUniqueKeyTest, TestLuceneMatchVersion, TestPhraseSuggestions, SynonymTokenizerTest, TestDFRSimilarityFactory, TestPerFieldSimilarity, TestLMJelinekMercerSimilarityFactory, ResourceLoaderTest, ScriptEngineTest, URLClassifyProcessorTest, TestFastLRUCache, DateMathParserTest, ChaosMonkeyNothingIsSafeTest, TestSuggestSpellingConverter, DateFieldTest, DOMUtilTest, TestSolrJ, UUIDFieldTest, FileUtilsTest, CircularListTest, DistributedMLTComponentTest, TestRTGBase, SolrTestCaseJ4Test, TestCrossCoreJoin, TestCursorMarkWithoutUniqueKey, TestHighlightDedupGrouping, TestEmbeddedSolrServerConstructors, TestJettySolrRunner, AliasIntegrationTest, CdcrBootstrapTest, CdcrVersionReplicationTest, CleanupOldIndexTest, CloudExitableDirectoryReaderTest, CollectionReloadTest, CollectionsAPISolrJTest, DeleteInactiveReplicaTest, DeleteLastCustomShardedReplicaTest, DeleteNodeTest, DeleteReplicaTest, LeaderFailoverAfterPartitionTest, LeaderInitiatedRecoveryOnShardRestartTest, SharedFSAutoReplicaFailoverTest, SimpleCollectionCreateDeleteTest, TestAuthenticationFramework, TestCloudDeleteByQuery, TestCloudPseudoReturnFields, TestCollectionAPI, TestConfigSetsAPI, TestConfigSetsAPIExclusivity, TestLeaderElectionZkExpiry, TestLeaderInitiatedRecoveryThread, TestLockTree, TestRandomFlRTGCloud, TestTolerantUpdateProcessorCloud, TestTolerantUpdateProcessorRandomCloud, TriLevelCompositeIdRoutingTest, HdfsRecoverLeaseTest, HdfsRestartWhileUpdatingTest, HdfsWriteToMultipleCollectionsTest, CachingDirectoryFactoryTest, RequestLoggingTest, TestConfigReload, CoreAdminCreateDiscoverTest, CoreAdminRequestStatusTest, InfoHandlerTest, SegmentsInfoRequestHandlerTest, DistributedFacetExistsSmallTest, DistributedFacetPivotLargeTest, DistributedFacetPivotLongTailTest]
   [junit4] Completed [532/631 (7!)] on J2 in 114.63s, 1 test, 1 error <<< FAILURES!

[...truncated 314 lines...]
   [junit4] JVM J2: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20160920_092105_519.sysout
   [junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid13092.hprof ...
   [junit4] Heap dump file created [484414264 bytes in 5.788 secs]
   [junit4] <<< JVM J2: EOF ----

[...truncated 11041 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:715: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid13092.hprof

Total time: 419 minutes 6 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1118 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1118/

18 tests failed.
FAILED:  org.apache.lucene.codecs.memory.TestDirectDocValuesFormat.testByteNumericsVsStoredFields

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([9FA0910DF6187A3D]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.codecs.memory.TestDirectDocValuesFormat

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([9FA0910DF6187A3D]:0)


FAILED:  org.apache.lucene.search.TestFuzzyQuery.testRandom

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([3ED1EDC594C2CDD4]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.search.TestFuzzyQuery

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([3ED1EDC594C2CDD4]:0)


FAILED:  org.apache.solr.TestDistributedSearch.test

Error Message:
Captured an uncaught exception in thread: Thread[id=50587, name=Thread-37029, state=RUNNABLE, group=TGRP-TestDistributedSearch]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=50587, name=Thread-37029, state=RUNNABLE, group=TGRP-TestDistributedSearch]
	at __randomizedtesting.SeedInfo.seed([320F8EB76642CB24:BA5BB16DC8BEA6DC]:0)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:39243//collection1: java.lang.NullPointerException
	at org.apache.solr.search.grouping.distributed.responseprocessor.StoredFieldsShardResponseProcessor.process(StoredFieldsShardResponseProcessor.java:41)
	at org.apache.solr.handler.component.QueryComponent.handleGroupedResponses(QueryComponent.java:751)
	at org.apache.solr.handler.component.QueryComponent.handleResponses(QueryComponent.java:734)
	at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:422)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:154)
	at org.apache.solr.core.SolrCore.execute(SolrCore.java:2107)
	at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:653)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:460)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(Thread.java:745)

	at __randomizedtesting.SeedInfo.seed([320F8EB76642CB24]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:608)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
	at org.apache.solr.TestDistributedSearch$1.run(TestDistributedSearch.java:1126)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationStartStop

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard1

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard1
	at __randomizedtesting.SeedInfo.seed([320F8EB76642CB24:B1CC5024729A40AD]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationStartStop(CdcrReplicationDistributedZkTest.java:167)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable

Error Message:
Timeout while trying to assert replication errors

Stack Trace:
java.lang.AssertionError: Timeout while trying to assert replication errors
	at __randomizedtesting.SeedInfo.seed([320F8EB76642CB24:D49BD4C46D1DA60C]:0)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable(CdcrReplicationDistributedZkTest.java:145)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.AssertionError
	at org.junit.Assert.fail(Assert.java:92)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertTrue(Assert.java:54)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable(CdcrReplicationDistributedZkTest.java:132)
	... 41 more


FAILED:  org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest.test

Error Message:
Error from server at http://127.0.0.1:55445/oneInstanceCollection24: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /oneInstanceCollection24/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.OutOfMemoryError},msg=SolrCore &apos;oneInstanceCollection24&apos; is not available due to init failure: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.,trace=org.apache.solr.common.SolrException: SolrCore &apos;oneInstanceCollection24&apos; is not available due to init failure: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1117)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:252)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:414)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.solr.common.SolrException: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.  at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:833)  at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:706)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:862)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:799)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:85)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:374)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:365)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:156)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:154)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:659)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:441)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)  ... 12 more Caused by: java.lang.RuntimeException: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.  at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:304)  at org.apache.solr.core.HdfsDirectoryFactory.getBlockDirectoryCache(HdfsDirectoryFactory.java:280)  at org.apache.solr.core.HdfsDirectoryFactory.create(HdfsDirectoryFactory.java:220)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:346)  at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:66)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:604)  at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:780)  ... 35 more Caused by: java.lang.OutOfMemoryError: Direct buffer memory  at java.nio.Bits.reserveMemory(Bits.java:693)  at java.nio.DirectByteBuffer.&lt;init&gt;(DirectByteBuffer.java:123)  at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)  at org.apache.solr.store.blockcache.BlockCache.&lt;init&gt;(BlockCache.java:63)  at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:302)  ... 41 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:55445/oneInstanceCollection24: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /oneInstanceCollection24/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.OutOfMemoryError},msg=SolrCore &apos;oneInstanceCollection24&apos; is not available due to init failure: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.,trace=org.apache.solr.common.SolrException: SolrCore &apos;oneInstanceCollection24&apos; is not available due to init failure: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1117)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:252)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:414)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.common.SolrException: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.
	at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:833)
	at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:706)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:862)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:799)
	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:85)
	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:374)
	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:365)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:156)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:154)
	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:659)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:441)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)
	... 12 more
Caused by: java.lang.RuntimeException: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.
	at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:304)
	at org.apache.solr.core.HdfsDirectoryFactory.getBlockDirectoryCache(HdfsDirectoryFactory.java:280)
	at org.apache.solr.core.HdfsDirectoryFactory.create(HdfsDirectoryFactory.java:220)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:346)
	at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:66)
	at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:604)
	at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:780)
	... 35 more
Caused by: java.lang.OutOfMemoryError: Direct buffer memory
	at java.nio.Bits.reserveMemory(Bits.java:693)
	at java.nio.DirectByteBuffer.&lt;init&gt;(DirectByteBuffer.java:123)
	at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)
	at org.apache.solr.store.blockcache.BlockCache.&lt;init&gt;(BlockCache.java:63)
	at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:302)
	... 41 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([320F8EB76642CB24:BA5BB16DC8BEA6DC]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:152)
	at org.apache.solr.cloud.BasicDistributedZkTest.testANewCollectionInOneInstanceWithManualShardAssignement(BasicDistributedZkTest.java:807)
	at org.apache.solr.cloud.BasicDistributedZkTest.test(BasicDistributedZkTest.java:362)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest

Error Message:
4 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest:     1) Thread[id=61761, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=61757, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=61763, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=61759, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 4 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest: 
   1) Thread[id=61761, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=61757, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=61763, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=61759, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([320F8EB76642CB24]:0)


FAILED:  org.apache.solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest.test

Error Message:
shard1 is not consistent.  Got 107 from http://127.0.0.1:50652/c/nt/collection1 (previous client) and got 105 from http://127.0.0.1:41516/c/nt/collection1

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 107 from http://127.0.0.1:50652/c/nt/collection1 (previous client) and got 105 from http://127.0.0.1:41516/c/nt/collection1
	at __randomizedtesting.SeedInfo.seed([320F8EB76642CB24:BA5BB16DC8BEA6DC]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1271)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:231)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest.test

Error Message:
Error from server at https://127.0.0.1:38529/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.OutOfMemoryError},msg=SolrCore &apos;collection1&apos; is not available due to init failure: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1117)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:252)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:414)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.solr.common.SolrException: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.  at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:833)  at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:706)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:862)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:518)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  ... 1 more Caused by: java.lang.RuntimeException: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.  at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:304)  at org.apache.solr.core.HdfsDirectoryFactory.getBlockDirectoryCache(HdfsDirectoryFactory.java:280)  at org.apache.solr.core.HdfsDirectoryFactory.create(HdfsDirectoryFactory.java:220)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:346)  at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:426)  at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:774)  ... 8 more Caused by: java.lang.OutOfMemoryError: Direct buffer memory  at java.nio.Bits.reserveMemory(Bits.java:693)  at java.nio.DirectByteBuffer.&lt;init&gt;(DirectByteBuffer.java:123)  at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)  at org.apache.solr.store.blockcache.BlockCache.&lt;init&gt;(BlockCache.java:63)  at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:302)  ... 13 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:38529/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.OutOfMemoryError},msg=SolrCore &apos;collection1&apos; is not available due to init failure: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1117)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:252)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:414)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.common.SolrException: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.
	at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:833)
	at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:706)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:862)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:518)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more
Caused by: java.lang.RuntimeException: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.
	at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:304)
	at org.apache.solr.core.HdfsDirectoryFactory.getBlockDirectoryCache(HdfsDirectoryFactory.java:280)
	at org.apache.solr.core.HdfsDirectoryFactory.create(HdfsDirectoryFactory.java:220)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:346)
	at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:426)
	at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:774)
	... 8 more
Caused by: java.lang.OutOfMemoryError: Direct buffer memory
	at java.nio.Bits.reserveMemory(Bits.java:693)
	at java.nio.DirectByteBuffer.&lt;init&gt;(DirectByteBuffer.java:123)
	at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)
	at org.apache.solr.store.blockcache.BlockCache.&lt;init&gt;(BlockCache.java:63)
	at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:302)
	... 13 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([320F8EB76642CB24:BA5BB16DC8BEA6DC]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.tryDelete(ChaosMonkeySafeLeaderTest.java:190)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:92)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:56594/ypv

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:56594/ypv
	at __randomizedtesting.SeedInfo.seed([320F8EB76642CB24:BA5BB16DC8BEA6DC]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.UnloadDistributedZkTest.testCoreUnloadAndLeaders(UnloadDistributedZkTest.java:349)
	at org.apache.solr.cloud.UnloadDistributedZkTest.test(UnloadDistributedZkTest.java:67)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 46 more


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigAliasReplication

Error Message:
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_320F8EB76642CB24-001/solr-instance-009/./collection1/data/index.20160919023959963, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_320F8EB76642CB24-001/solr-instance-009/./collection1/data, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_320F8EB76642CB24-001/solr-instance-009/./collection1/data/snapshot_metadata, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_320F8EB76642CB24-001/solr-instance-009/./collection1/data/index.20160919024000125] expected:<3> but was:<4>

Stack Trace:
java.lang.AssertionError: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_320F8EB76642CB24-001/solr-instance-009/./collection1/data/index.20160919023959963, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_320F8EB76642CB24-001/solr-instance-009/./collection1/data, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_320F8EB76642CB24-001/solr-instance-009/./collection1/data/snapshot_metadata, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_320F8EB76642CB24-001/solr-instance-009/./collection1/data/index.20160919024000125] expected:<3> but was:<4>
	at __randomizedtesting.SeedInfo.seed([320F8EB76642CB24:C57C60EFA0AA64C2]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.apache.solr.handler.TestReplicationHandler.checkForSingleIndex(TestReplicationHandler.java:902)
	at org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigAliasReplication(TestReplicationHandler.java:1334)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard1

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard1
	at __randomizedtesting.SeedInfo.seed([320F8EB76642CB24:6E126936A8627F12]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart(CdcrReplicationDistributedZkTest.java:264)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:52019

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:52019
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:516)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterLeaderChange

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard2

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard2
	at __randomizedtesting.SeedInfo.seed([320F8EB76642CB24:E0FFC25438ED6D16]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterLeaderChange(CdcrReplicationDistributedZkTest.java:306)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evalu

[...truncated too long message...]

odAccessorImpl.java:62)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.MethodMetric$2.snapshot(MethodMetric.java:111)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.MethodMetric.snapshot(MethodMetric.java:144)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.MetricsRegistry.snapshot(MetricsRegistry.java:401)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.MetricsSourceBuilder$1.getMetrics(MetricsSourceBuilder.java:79)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMetrics(MetricsSourceAdapter.java:194)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.updateJmxCache(MetricsSourceAdapter.java:172)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMBeanInfo(MetricsSourceAdapter.java:151)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getClassName(DefaultMBeanServerInterceptor.java:1804)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.safeGetClassName(DefaultMBeanServerInterceptor.java:1595)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.checkMBeanPermission(DefaultMBeanServerInterceptor.java:1813)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:430)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
   [junit4]   2> 	at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
   [junit4]   2> 	at org.apache.hadoop.metrics2.util.MBeans.unregister(MBeans.java:81)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stopMBeans(MetricsSourceAdapter.java:226)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stop(MetricsSourceAdapter.java:211)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stopSources(MetricsSystemImpl.java:463)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
   [junit4]   2> 	at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
   [junit4]   2> 	at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
   [junit4]   2> 	at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:198)
   [junit4]   2> 	at org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest.teardownClass(HdfsChaosMonkeySafeLeaderTest.java:46)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: java.lang.NullPointerException
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
   [junit4]   2> 	... 54 more
   [junit4]   2> 6626859 INFO  (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[320F8EB76642CB24]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_320F8EB76642CB24-001
   [junit4]   2> Sep 18, 2016 7:58:00 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62), sim=RandomSimilarity(queryNorm=false): {}, locale=sl-SI, timezone=Pacific/Tahiti
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=2,free=319403528,total=524812288
   [junit4]   2> NOTE: All tests run in this JVM: [ZkCLITest, SliceStateTest, HdfsRecoverLeaseTest, CdcrVersionReplicationTest, TestHdfsBackupRestoreCore, PKIAuthenticationIntegrationTest, HdfsDirectoryFactoryTest, DistribDocExpirationUpdateProcessorTest, DocumentBuilderTest, TestHashPartitioner, HdfsDirectoryTest, TestRestoreCore, TermVectorComponentTest, HighlighterMaxOffsetTest, ReplaceNodeTest, SampleTest, DocValuesMissingTest, TestManagedSchemaAPI, TestElisionMultitermQuery, HighlighterTest, TestFileDictionaryLookup, TestCSVResponseWriter, TestMacros, CdcrReplicationDistributedZkTest, AliasIntegrationTest, SuggestComponentTest, TestDefaultSearchFieldResource, DistributedQueryComponentCustomSortTest, OpenCloseCoreStressTest, TestUseDocValuesAsStored2, SearchHandlerTest, SolrIndexConfigTest, DistributedIntervalFacetingTest, TestStandardQParsers, TestGraphMLResponseWriter, BigEndianAscendingWordSerializerTest, TestQueryUtils, BJQParserTest, TestEmbeddedSolrServerConstructors, TestOverriddenPrefixQueryForCustomFieldType, TestStressVersions, TestFieldCacheWithThreads, TestConfigOverlay, TestSurroundQueryParser, TestManagedResource, TestCloudInspectUtil, BlobRepositoryCloudTest, ZkSolrClientTest, SpatialFilterTest, ShardRoutingCustomTest, FullHLLTest, CloudExitableDirectoryReaderTest, HdfsUnloadDistributedZkTest, TestSearchPerf, TermsComponentTest, MoreLikeThisHandlerTest, SimplePostToolTest, URLClassifyProcessorTest, ClassificationUpdateProcessorFactoryTest, TestBulkSchemaAPI, SpellCheckCollatorTest, HdfsBasicDistributedZk2Test, TestSuggestSpellingConverter, MultiTermTest, SystemInfoHandlerTest, TestManagedStopFilterFactory, TestClusterStateMutator, TestShortCircuitedRequests, TestHashQParserPlugin, RollingRestartTest, ShardSplitTest, TestRTGBase, BlockJoinFacetSimpleTest, TestSchemaSimilarityResource, TestCursorMarkWithoutUniqueKey, HdfsSyncSliceTest, BasicFunctionalityTest, TestWordDelimiterFilterFactory, TestCharFilters, TestUpdate, TestFieldSortValues, PeerSyncReplicationTest, DistanceUnitsTest, TestCollectionAPI, VersionInfoTest, TestSolrCloudWithKerberosAlt, SolrIndexSplitterTest, TestExactStatsCache, TestInfoStreamLogging, DeleteLastCustomShardedReplicaTest, TestRuleBasedAuthorizationPlugin, TestDocBasedVersionConstraints, TestFieldCacheSort, TestLRUCache, SimpleCollectionCreateDeleteTest, TestCloudSchemaless, BooleanFieldTest, CreateCollectionCleanupTest, TestSweetSpotSimilarityFactory, TestRequestForwarding, BlockCacheTest, TestScoreJoinQPScore, SolrCmdDistributorTest, TestDistribDocBasedVersion, TestLockTree, TestMiniSolrCloudCluster, TestRequestStatusCollectionAPI, TestStressCloudBlindAtomicUpdates, VMParamsZkACLAndCredentialsProvidersTest, HdfsBasicDistributedZkTest, HdfsChaosMonkeyNothingIsSafeTest, HdfsChaosMonkeySafeLeaderTest]
   [junit4] Completed [502/631 (9!)] on J1 in 70.57s, 1 test, 1 error <<< FAILURES!

[...truncated 60608 lines...]



[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1117 - Still unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1117/

6 tests failed.
FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard1

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard1
	at __randomizedtesting.SeedInfo.seed([937EBF689F1DDA9C:CF6358E9513D6EAA]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart(CdcrReplicationDistributedZkTest.java:264)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:53419/j_xez/ar

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:53419/j_xez/ar
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:516)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterLeaderChange

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard2

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard2
	at __randomizedtesting.SeedInfo.seed([937EBF689F1DDA9C:418EF38BC1B27CAE]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterLeaderChange(CdcrReplicationDistributedZkTest.java:306)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:40723/pyaha

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:40723/pyaha
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:516)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
	at __randomizedtesting.SeedInfo.seed([937EBF689F1DDA9C]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.GeneratedMethodAccessor22.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest:     1) Thread[id=132659, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=132699, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=132704, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=132664, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=132674, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=132679, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=132684, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=132669, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=132694, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=132689, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 
   1) Thread[id=132659, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=132699, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=132704, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=132664, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=132674, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=132679, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=132684, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=132669, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=132694, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=132689, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([937EBF689F1DDA9C]:0)




Build Log:
[...truncated 11697 lines...]
   [junit4] Suite: org.apache.solr.cloud.CdcrReplicationDistributedZkTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/init-core-data-001
   [junit4]   2> 501069 INFO  (SUITE-CdcrReplicationDistributedZkTest-seed#[937EBF689F1DDA9C]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 501069 INFO  (SUITE-CdcrReplicationDistributedZkTest-seed#[937EBF689F1DDA9C]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /hl/xk
   [junit4]   2> 501090 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 501095 INFO  (Thread-4194) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 501095 INFO  (Thread-4194) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 501197 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.ZkTestServer start zk server on port:54824
   [junit4]   2> 501211 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 501215 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 501229 INFO  (zkCallback-211-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@114a745b name:ZooKeeperConnection Watcher:127.0.0.1:54824 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 501229 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 501230 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 501230 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 501233 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 501234 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 501251 INFO  (zkCallback-212-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@27746dde name:ZooKeeperConnection Watcher:127.0.0.1:54824/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 501251 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 501252 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 501252 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1
   [junit4]   2> 501256 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/shards
   [junit4]   2> 501257 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection
   [junit4]   2> 501258 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection/shards
   [junit4]   2> 501260 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-cdcr.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 501260 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 501261 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 501262 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/schema.xml
   [junit4]   2> 501267 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 501268 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 501269 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 501269 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 501270 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 501270 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/protwords.txt
   [junit4]   2> 501272 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 501272 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/currency.xml
   [junit4]   2> 501273 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 501273 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 501279 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 501280 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 501281 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 501281 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 501282 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 501282 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 501284 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 501284 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 501288 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testBatchAddsWithDelete
   [junit4]   2> 501408 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores/collection1
   [junit4]   2> 501409 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 501412 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5282d201{/hl/xk,null,AVAILABLE}
   [junit4]   2> 501413 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.e.j.s.ServerConnector Started ServerConnector@4082e43a{HTTP/1.1,[http/1.1]}{127.0.0.1:43328}
   [junit4]   2> 501413 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.e.j.s.Server Started @510644ms
   [junit4]   2> 501413 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {shards=shard1, hostContext=/hl/xk, hostPort=43328, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores}
   [junit4]   2> 501414 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 501414 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001'
   [junit4]   2> 501414 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 501414 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 501414 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 501415 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 501427 INFO  (zkCallback-213-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@52ce4a5a name:ZooKeeperConnection Watcher:127.0.0.1:54824/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 501427 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 501427 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 501429 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 501429 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/solr.xml
   [junit4]   2> 501434 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores
   [junit4]   2> 501434 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.CoreContainer New CoreContainer 2039324192
   [junit4]   2> 501435 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001]
   [junit4]   2> 501435 WARN  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.CoreContainer Couldn't add files from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/lib to classpath: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/lib
   [junit4]   2> 501447 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,
   [junit4]   2> 501455 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.u.UpdateShardHandler Created UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&maxConnections=100000&maxConnectionsPerHost=100000
   [junit4]   2> 501455 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 501455 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 501455 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:54824/solr
   [junit4]   2> 501455 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 501455 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 501463 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 501479 INFO  (zkCallback-216-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3ba14d60 name:ZooKeeperConnection Watcher:127.0.0.1:54824 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 501495 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 501495 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 501507 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$12/1549056544@2ff19dd2
   [junit4]   2> 501508 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 501516 INFO  (zkCallback-217-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4add716 name:ZooKeeperConnection Watcher:127.0.0.1:54824/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 501516 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 501520 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
   [junit4]   2> 501521 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
   [junit4]   2> 501524 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
   [junit4]   2> 501525 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
   [junit4]   2> 501527 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
   [junit4]   2> 501528 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 501529 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 501530 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 501538 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 501539 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 501539 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 501540 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 501542 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
   [junit4]   2> 501543 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
   [junit4]   2> 501544 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 501544 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/96600495612559364-127.0.0.1:43328_hl%2Fxk-n_0000000000
   [junit4]   2> 501545 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:43328_hl%2Fxk
   [junit4]   2> 501545 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
   [junit4]   2> 501546 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.Overseer Overseer (id=96600495612559364-127.0.0.1:43328_hl%2Fxk-n_0000000000) starting
   [junit4]   2> 501547 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
   [junit4]   2> 501556 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=10000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 501575 INFO  (OverseerStateUpdate-96600495612559364-127.0.0.1:43328_hl%2Fxk-n_0000000000) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.Overseer Starting to work on the main queue
   [junit4]   2> 501576 INFO  (OverseerCollectionConfigSetProcessor-96600495612559364-127.0.0.1:43328_hl%2Fxk-n_0000000000) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
   [junit4]   2> 501586 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43328_hl%2Fxk
   [junit4]   2> 501586 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:43328_hl%2Fxk
   [junit4]   2> 501587 INFO  (OverseerStateUpdate-96600495612559364-127.0.0.1:43328_hl%2Fxk-n_0000000000) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 501595 INFO  (zkCallback-217-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 501596 INFO  (zkCallback-217-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 501637 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 501637 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 501638 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores
   [junit4]   2> 501640 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.json, collection=tmp_collection, coreNodeName=, dataDir=data/, shard=}
   [junit4]   2> 501640 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.CorePropertiesLocator Found core collection1 in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores/collection1
   [junit4]   2> 501640 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions
   [junit4]   2> 501643 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.ZkController publishing state=down
   [junit4]   2> 501643 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.ZkController numShards not found on descriptor - reading it from system property
   [junit4]   2> 501645 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.ZkController Collection tmp_collection not visible yet, but flagging it so a watch is registered when it becomes visible
   [junit4]   2> 501646 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.c.ZkStateReader Deleting data for [tmp_collection]
   [junit4]   2> 501647 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.s.SolrDispatchFilter user.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1
   [junit4]   2> 501647 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 501648 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.ZkController look for our core node name
   [junit4]   2> 501863 INFO  (OverseerStateUpdate-96600495612559364-127.0.0.1:43328_hl%2Fxk-n_0000000000) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:43328/hl/xk",
   [junit4]   2>   "node_name":"127.0.0.1:43328_hl%2Fxk",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":null,
   [junit4]   2>   "collection":"tmp_collection",
   [junit4]   2>   "operation":"state"} current state version: 0
   [junit4]   2> 501863 INFO  (OverseerStateUpdate-96600495612559364-127.0.0.1:43328_hl%2Fxk-n_0000000000) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.o.ReplicaMutator Update state numShards=2 message={
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:43328/hl/xk",
   [junit4]   2>   "node_name":"127.0.0.1:43328_hl%2Fxk",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":null,
   [junit4]   2>   "collection":"tmp_collection",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 501866 INFO  (OverseerStateUpdate-96600495612559364-127.0.0.1:43328_hl%2Fxk-n_0000000000) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.o.ClusterStateMutator building a new cName: tmp_collection
   [junit4]   2> 501866 INFO  (OverseerStateUpdate-96600495612559364-127.0.0.1:43328_hl%2Fxk-n_0000000000) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
   [junit4]   2> 501871 INFO  (zkCallback-217-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 502651 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.ZkController waiting to find shard id in clusterstate for collection1
   [junit4]   2> 502651 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.ZkController Check for collection zkNode:tmp_collection
   [junit4]   2> 502651 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.ZkController Creating collection in ZooKeeper:tmp_collection
   [junit4]   2> 502652 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.ZkController Looking for collection configName
   [junit4]   2> 502652 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.ZkController Only one config set found in zk - using it:conf1
   [junit4]   2> 502652 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.c.SolrZkClient makePath: /collections/tmp_collection
   [junit4]   2> 502653 INFO  (zkCallback-217-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 502654 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.c.ZkStateReader Load collection config from: [/collections/tmp_collection]
   [junit4]   2> 502654 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.c.ZkStateReader path=[/collections/tmp_collection] [configName]=[conf1] specified config exists in ZooKeeper
   [junit4]   2> 502654 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores/collection1'
   [junit4]   2> 502654 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 502654 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 502669 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.Config loaded config solrconfig.xml with version 0 
   [junit4]   2> 502692 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 502695 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 502727 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 502732 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 502812 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.s.IndexSchema default search field in schema is text
   [junit4]   2> 502813 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 502814 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.s.FileExchangeRateProvider Reloading exchange rates from file currency.xml
   [junit4]   2> 502818 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.s.FileExchangeRateProvider Reloading exchange rates from file currency.xml
   [junit4]   2> 502841 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properties: Can't find resource 'configsetprops.json' in classpath or '/configs/conf1', cwd=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1
   [junit4]   2> 502841 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection tmp_collection
   [junit4]   2> 502841 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 502841 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores/collection1/data/]
   [junit4]   2> 502841 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@372ca662
   [junit4]   2> 502842 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores/collection1/data/snapshot_metadata
   [junit4]   2> 502842 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.s.SolrSnapshotMetaDataManager Loading from snapshot metadata file...
   [junit4]   2> 502842 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores/collection1/data
   [junit4]   2> 502842 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore New index directory detected: old=null new=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores/collection1/data/index/
   [junit4]   2> 502842 WARN  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Solr index directory '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 502842 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores/collection1/data/index
   [junit4]   2> 502843 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=46, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1387051555292735]
   [junit4]   2> 502843 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@73f11028 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@36427ede; maxCacheMB=0.205078125 maxMergeSizeMB=0.943359375)),segFN=segments_1,generation=1}
   [junit4]   2> 502843 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 502846 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "cdcr-processor-chain"
   [junit4]   2> 502857 INFO  (OldIndexDirectoryCleanupThreadForCore-collection1) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore Looking for old index directories to cleanup for core collection1 in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores/collection1/data/
   [junit4]   2> 502857 WARN  (OldIndexDirectoryCleanupThreadForCore-collection1) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.DirectoryFactory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores/collection1/data/ does not point to a valid data directory; skipping clean-up of old index directories.
   [junit4]   2> 502866 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 502867 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 502868 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 502869 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 502870 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 502882 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,standard,/update/json/docs,/admin/luke,/export,/get,/admin/properties,/update/json,/admin/threads,/update/csv,/sql,/graph,/admin/segments,/admin/system,/replication,/config,/stream,/schema,/admin/plugins,/admin/logging,/admin/ping,/cdcr,/update,/admin/file,/terms
   [junit4]   2> 502882 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 502894 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.CdcrUpdateLog
   [junit4]   2> 502894 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 502894 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 502894 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 502895 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1721589593, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 502899 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@73f11028 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@36427ede; maxCacheMB=0.205078125 maxMergeSizeMB=0.943359375)),segFN=segments_1,generation=1}
   [junit4]   2> 502899 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 502900 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@301d3483[collection1] main]
   [junit4]   2> 502900 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.c.ZkStateReader Load collection config from: [/collections/tmp_collection]
   [junit4]   2> 502901 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.c.ZkStateReader path=[/collections/tmp_collection] [configName]=[conf1] specified config exists in ZooKeeper
   [junit4]   2> 502901 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 502901 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 502901 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.r.RestManager Initializing RestManager with initArgs: {}
   [junit4]   2> 502901 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 502902 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 502902 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 502902 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 502902 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 502907 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.c.SolrZkClient makePath: /collections/tmp_collection/cdcr/state
   [junit4]   2> 502909 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.h.CdcrBufferStateManager Created znode /collections/tmp_collection/cdcr/state/buffer
   [junit4]   2> 502914 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.h.CdcrProcessStateManager Created znode /collections/tmp_collection/cdcr/state/process
   [junit4]   2> 503014 INFO  (searcherExecutor-518-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk x:collection1 s:shard2 c:tmp_collection r:core_node1) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@301d3483[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 503015 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateLog Looking up max value of version field to seed version buckets
   [junit4]   2> 503015 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.u.VersionInfo Refreshing highest value of _version_ for 65536 version buckets from index
   [junit4]   2> 503015 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.u.VersionInfo No terms found for _version_, cannot seed version bucket highest value from index
   [junit4]   2> 503015 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1545607931789049856
   [junit4]   2> 503017 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateLog Took 1.0ms to seed version buckets with highest version 1545607931789049856
   [junit4]   2> 503017 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.ZkController watch zkdir /configs/conf1
   [junit4]   2> 503020 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.CoreContainer registering core: collection1
   [junit4]   2> 503027 INFO  (coreZkRegister-510-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk x:collection1 s:shard2 c:tmp_collection r:core_node1) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.ZkController Register replica - core:collection1 address:http://127.0.0.1:43328/hl/xk collection:tmp_collection shard:shard2
   [junit4]   2> 503035 INFO  (coreZkRegister-510-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk x:collection1 s:shard2 c:tmp_collection r:core_node1) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.c.SolrZkClient makePath: /collections/tmp_collection/leader_elect/shard2/election
   [junit4]   2> 503087 INFO  (coreZkRegister-510-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk x:collection1 s:shard2 c:tmp_collection r:core_node1) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.LeaderElector Joined leadership election with path: /collections/tmp_collection/leader_elect/shard2/election/96600495612559364-core_node1-n_0000000000
   [junit4]   2> 503091 INFO  (coreZkRegister-510-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk x:collection1 s:shard2 c:tmp_collection r:core_node1) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Running the leader process for shard=shard2 and weAreReplacement=false and leaderVoteWait=10000
   [junit4]   2> 503106 INFO  (coreZkRegister-510-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk x:collection1 s:shard2 c:tmp_collection r:core_node1) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 503106 INFO  (coreZkRegister-510-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk x:collection1 s:shard2 c:tmp_collection r:core_node1) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 503106 INFO  (coreZkRegister-510-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk x:collection1 s:shard2 c:tmp_collection r:core_node1) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:43328/hl/xk/collection1/
   [junit4]   2> 503106 INFO  (coreZkRegister-510-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk x:collection1 s:shard2 c:tmp_collection r:core_node1) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 503106 INFO  (OverseerStateUpdate-96600495612559364-127.0.0.1:43328_hl%2Fxk-n_0000000000) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "operation":"leader",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "collection":"tmp_collection"} current state version: 1
   [junit4]   2> 503106 INFO  (coreZkRegister-510-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk x:collection1 s:shard2 c:tmp_collection r:core_node1) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 

[...truncated too long message...]

unit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
   [junit4]   2> 	at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
   [junit4]   2> 	at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
   [junit4]   2> 	at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:198)
   [junit4]   2> 	at org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.teardownClass(HdfsCollectionsAPIDistributedZkTest.java:52)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: java.lang.NullPointerException
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
   [junit4]   2> 	... 54 more
   [junit4]   2> 8304437 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[937EBF689F1DDA9C]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_937EBF689F1DDA9C-001
   [junit4]   2> Sep 16, 2016 8:26:29 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {other_tl1=Lucene50(blocksize=128), range_facet_l_dv=Lucene50(blocksize=128), _version_=Lucene50(blocksize=128), multiDefault=PostingsFormat(name=MockRandom), a_t=PostingsFormat(name=MockRandom), intDefault=Lucene50(blocksize=128), a_i1=PostingsFormat(name=MockRandom), id=Lucene50(blocksize=128), range_facet_i_dv=PostingsFormat(name=MockRandom), text=PostingsFormat(name=LuceneFixedGap), range_facet_l=PostingsFormat(name=MockRandom), timestamp=PostingsFormat(name=MockRandom)}, docValues:{range_facet_l_dv=DocValuesFormat(name=Lucene54), range_facet_i_dv=DocValuesFormat(name=Memory), timestamp=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=1608, maxMBSortInHeap=7.964928059876776, sim=ClassicSimilarity, locale=ro, timezone=Etc/GMT+11
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=240799904,total=516423680
   [junit4]   2> NOTE: All tests run in this JVM: [ClusterStateTest, TestSolrConfigHandler, HdfsRestartWhileUpdatingTest, ZkNodePropsTest, TestStressCloudBlindAtomicUpdates, TestReplicationHandler, TestSolrDeletionPolicy2, DistributedExpandComponentTest, OutputWriterTest, SmileWriterTest, DistributedSpellCheckComponentTest, TestTolerantUpdateProcessorRandomCloud, FastVectorHighlighterTest, HdfsChaosMonkeyNothingIsSafeTest, PolyFieldTest, TestMacroExpander, TestRecoveryHdfs, TestFieldCacheVsDocValues, TestHdfsCloudBackupRestore, TestHighlightDedupGrouping, TestAtomicUpdateErrorCases, TestRandomCollapseQParserPlugin, RankQueryTest, SecurityConfHandlerTest, RestartWhileUpdatingTest, TestNoOpRegenerator, TestSolrCloudWithDelegationTokens, StandardRequestHandlerTest, BufferStoreTest, CollectionsAPISolrJTest, TestPseudoReturnFields, TestRandomFlRTGCloud, ConnectionReuseTest, TestFunctionQuery, ReturnFieldsTest, QueryElevationComponentTest, TestFieldCacheSanityChecker, AlternateDirectoryTest, AtomicUpdatesTest, CdcrRequestHandlerTest, TestRangeQuery, ExitableDirectoryReaderTest, CdcrBootstrapTest, DistributedQueueTest, TestSSLRandomization, AnalyticsQueryTest, DistributedFacetPivotLongTailTest, TestFoldingMultitermQuery, BlockJoinFacetRandomTest, PeerSyncTest, TestScoreJoinQPNoScore, TestHdfsUpdateLog, ScriptEngineTest, TestFastWriter, TestFastLRUCache, TestRandomDVFaceting, AddBlockUpdateTest, OverseerModifyCollectionTest, TestTrieFacet, BadCopyFieldTest, QueryEqualityTest, CollectionStateFormat2Test, TestExpandComponent, TestManagedSchema, SolrCloudExampleTest, AnalysisAfterCoreReloadTest, TestComponentsName, TestDistributedSearch, TestRealTimeGet, TestJoin, UUIDFieldTest, DeleteReplicaTest, StressHdfsTest, TestQuerySenderNoQuery, TestSchemaResource, SolrRequestParserTest, TestRTimerTree, TestBinaryResponseWriter, TestCodecSupport, TestLocalFSCloudBackupRestore, NumericFieldsTest, CurrencyFieldOpenExchangeTest, DocumentAnalysisRequestHandlerTest, TestSolr4Spatial, SignatureUpdateProcessorFactoryTest, TestRawTransformer, BasicZkTest, HLLUtilTest, DistribCursorPagingTest, TestCloudDeleteByQuery, RemoteQueryErrorTest, JavabinLoaderTest, TestSolrCoreSnapshots, TestConfigSetProperties, SuggestComponentContextFilterQueryTest, DistributedFacetPivotWhiteBoxTest, TestRestManager, PreAnalyzedUpdateProcessorTest, TestJmxIntegration, TestMergePolicyConfig, TestDocSet, TestBinaryField, MinimalSchemaTest, TestFuzzyAnalyzedSuggestions, ExternalFileFieldSortTest, TestSolrCoreProperties, DirectSolrConnectionTest, NotRequiredUniqueKeyTest, TestPhraseSuggestions, SpellPossibilityIteratorTest, TestCharFilters, SynonymTokenizerTest, TestPerFieldSimilarity, TestLMDirichletSimilarityFactory, TestIBSimilarityFactory, TestJmxMonitoredMap, ResourceLoaderTest, PluginInfoTest, ChaosMonkeyNothingIsSafeTest, PrimUtilsTest, TestSuggestSpellingConverter, DateFieldTest, DOMUtilTest, TestLRUCache, TestDocumentBuilder, SliceStateTest, DistributedMLTComponentTest, TestRTGBase, CursorPagingTest, TestCrossCoreJoin, TestSimpleTrackingShardHandler, ActionThrottleTest, AliasIntegrationTest, AsyncCallRequestStatusResponseTest, BaseCdcrDistributedZkTest, CdcrReplicationHandlerTest, CdcrVersionReplicationTest, CleanupOldIndexTest, CollectionTooManyReplicasTest, CollectionsAPIAsyncDistributedZkTest, ConnectionManagerTest, DeleteInactiveReplicaTest, LeaderFailoverAfterPartitionTest, LeaderInitiatedRecoveryOnCommitTest, LeaderInitiatedRecoveryOnShardRestartTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverseerCollectionConfigSetProcessorTest, OverseerTaskQueueTest, PeerSyncReplicationTest, ReplaceNodeTest, ReplicationFactorTest, SaslZkACLProviderTest, SharedFSAutoReplicaFailoverUtilsTest, SimpleCollectionCreateDeleteTest, SolrCLIZkUtilsTest, TestCloudPivotFacet, TestCloudPseudoReturnFields, TestCollectionAPI, TestConfigSetsAPI, TestConfigSetsAPIExclusivity, TestConfigSetsAPIZkFailure, TestCryptoKeys, TestDistribDocBasedVersion, TestExclusionRuleCollectionAccess, TestLeaderInitiatedRecoveryThread, TestOnReconnectListenerSupport, TestRequestForwarding, TestShortCircuitedRequests, TestSolrCloudWithKerberosAlt, VMParamsZkACLAndCredentialsProvidersTest, HdfsBasicDistributedZk2Test, HdfsChaosMonkeySafeLeaderTest, HdfsCollectionsAPIDistributedZkTest]
   [junit4] Completed [523/631 (4!)] on J0 in 1419.64s, 1 test, 1 error <<< FAILURES!

[...truncated 60548 lines...]



[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1116 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1116/

11 tests failed.
FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testOps

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard2

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard2
	at __randomizedtesting.SeedInfo.seed([11395C375D5ACE69:D41C0D41269C5B4]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testOps(CdcrReplicationDistributedZkTest.java:464)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterLeaderChange

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard1

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard1
	at __randomizedtesting.SeedInfo.seed([11395C375D5ACE69:C3C910D403F5685B]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterLeaderChange(CdcrReplicationDistributedZkTest.java:305)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:43481

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:43481
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:516)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  org.apache.solr.cloud.CollectionsAPISolrJTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:56799

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:56799
	at __randomizedtesting.SeedInfo.seed([11395C375D5ACE69:996D63EDF3A6A391]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:415)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:367)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1280)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1050)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:992)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:166)
	at org.apache.solr.cloud.CollectionsAPISolrJTest.testAddAndDeleteReplica(CollectionsAPISolrJTest.java:310)
	at org.apache.solr.cloud.CollectionsAPISolrJTest.test(CollectionsAPISolrJTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 52 more


FAILED:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=168560, name=Thread-29390, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=168560, name=Thread-29390, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:57528/chvf/xm/collection1
	at __randomizedtesting.SeedInfo.seed([11395C375D5ACE69]:0)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:644)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:57528/chvf/xm/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:166)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:642)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 5 more


FAILED:  org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit

Error Message:
expected:<1> but was:<2>

Stack Trace:
java.lang.AssertionError: expected:<1> but was:<2>
	at __randomizedtesting.SeedInfo.seed([11395C375D5ACE69:E874CF98612F83E3]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit(ShardSplitTest.java:278)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail

Error Message:
expected:<200> but was:<404>

Stack Trace:
java.lang.AssertionError: expected:<200> but was:<404>
	at __randomizedtesting.SeedInfo.seed([11395C375D5ACE69:7986691D8DC0DC85]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.cancelDelegationToken(TestSolrCloudWithDelegationTokens.java:137)
	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail(TestSolrCloudWithDelegationTokens.java:282)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest.test

Error Message:
There are still nodes recoverying - waited for 440 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 440 seconds
	at __randomizedtesting.SeedInfo.seed([11395C375D5ACE69:996D63EDF3A6A391]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:182)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:862)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1418)
	at org.apache.solr.cloud.TlogReplayBufferedWhileIndexingTest.test(TlogReplayBufferedWhileIndexingTest.java:99)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.handler.TestReplicationHandler

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory]
	at __randomizedtesting.SeedInfo.seed([11395C375D5ACE69]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.GeneratedMethodAccessor21.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
	at __randomizedtesting.SeedInfo.seed([11395C375D5ACE69]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.GeneratedMethodAccessor95.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest:     1) Thread[id=80041, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=80021, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=80001, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=80006, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=80011, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=80026, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=80016, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=80036, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=80031, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=80046, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 
   1) Thread[id=80041, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=80021, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=80001, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=80006, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=80011, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=80026, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=80016, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=80036, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=80031, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=80046, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([11395C375D5ACE69]:0)




Build Log:
[...truncated 11203 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithDelegationTokens
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_11395C375D5ACE69-001/init-core-data-001
   [junit4]   2> 379422 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[11395C375D5ACE69]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 379432 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[11395C375D5ACE69]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 379436 INFO  (Thread-559) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 379436 INFO  (Thread-559) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 379536 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[11395C375D5ACE69]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:52735
   [junit4]   2> 379536 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[11395C375D5ACE69]-worker) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 379540 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[11395C375D5ACE69]-worker) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 379584 INFO  (zkCallback-311-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@9b6aee5 name:ZooKeeperConnection Watcher:127.0.0.1:52735 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 379584 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[11395C375D5ACE69]-worker) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 379584 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[11395C375D5ACE69]-worker) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 379584 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[11395C375D5ACE69]-worker) [    ] o.a.s.c.c.SolrZkClient makePath: /solr/solr.xml
   [junit4]   2> 379601 INFO  (jetty-launcher-310-thread-1) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 379601 INFO  (jetty-launcher-310-thread-2) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 379602 INFO  (jetty-launcher-310-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@767e9948{/solr,null,AVAILABLE}
   [junit4]   2> 379603 INFO  (jetty-launcher-310-thread-1) [    ] o.e.j.s.ServerConnector Started ServerConnector@64e13eb7{HTTP/1.1,[http/1.1]}{127.0.0.1:59036}
   [junit4]   2> 379603 INFO  (jetty-launcher-310-thread-1) [    ] o.e.j.s.Server Started @387191ms
   [junit4]   2> 379603 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=59036}
   [junit4]   2> 379603 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 379603 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_11395C375D5ACE69-001/tempDir-001/node1'
   [junit4]   2> 379603 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 379603 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 379604 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 379616 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 379719 INFO  (jetty-launcher-310-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@e3dec11{/solr,null,AVAILABLE}
   [junit4]   2> 379719 INFO  (jetty-launcher-310-thread-2) [    ] o.e.j.s.ServerConnector Started ServerConnector@11107aae{HTTP/1.1,[http/1.1]}{127.0.0.1:46719}
   [junit4]   2> 379719 INFO  (jetty-launcher-310-thread-2) [    ] o.e.j.s.Server Started @387308ms
   [junit4]   2> 379720 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=46719}
   [junit4]   2> 379720 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 379720 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_11395C375D5ACE69-001/tempDir-001/node2'
   [junit4]   2> 379720 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 379720 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 379720 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 379920 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 379948 INFO  (zkCallback-312-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@7225043d name:ZooKeeperConnection Watcher:127.0.0.1:52735/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 379948 INFO  (zkCallback-313-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@365e98ff name:ZooKeeperConnection Watcher:127.0.0.1:52735/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 379952 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 379952 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 379952 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 379952 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 379956 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 380138 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 380144 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_11395C375D5ACE69-001/tempDir-001/node2/.
   [junit4]   2> 380168 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.CoreContainer New CoreContainer 416722465
   [junit4]   2> 380168 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_11395C375D5ACE69-001/tempDir-001/node2]
   [junit4]   2> 380168 WARN  (jetty-launcher-310-thread-2) [    ] o.a.s.c.CoreContainer Couldn't add files from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_11395C375D5ACE69-001/tempDir-001/node2/lib to classpath: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_11395C375D5ACE69-001/tempDir-001/node2/lib
   [junit4]   2> 380168 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,
   [junit4]   2> 380169 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_11395C375D5ACE69-001/tempDir-001/node1/.
   [junit4]   2> 380169 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.CoreContainer New CoreContainer 1273173647
   [junit4]   2> 380170 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_11395C375D5ACE69-001/tempDir-001/node1]
   [junit4]   2> 380170 WARN  (jetty-launcher-310-thread-1) [    ] o.a.s.c.CoreContainer Couldn't add files from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_11395C375D5ACE69-001/tempDir-001/node1/lib to classpath: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_11395C375D5ACE69-001/tempDir-001/node1/lib
   [junit4]   2> 380170 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,
   [junit4]   2> 380171 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.u.UpdateShardHandler Created UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&maxConnections=100000&maxConnectionsPerHost=100000
   [junit4]   2> 380171 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 380172 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 380172 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52735/solr
   [junit4]   2> 380172 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 380172 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 380172 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 380177 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.u.UpdateShardHandler Created UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&maxConnections=100000&maxConnectionsPerHost=100000
   [junit4]   2> 380180 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 380180 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 380180 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52735/solr
   [junit4]   2> 380180 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 380180 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 380209 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 380240 INFO  (zkCallback-319-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6880baf2 name:ZooKeeperConnection Watcher:127.0.0.1:52735 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 380260 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 380261 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 380262 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$90/738729067@6c0cd33b
   [junit4]   2> 380276 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 380286 INFO  (zkCallback-316-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@687cb563 name:ZooKeeperConnection Watcher:127.0.0.1:52735 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 380286 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 380287 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 380323 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$90/738729067@3a58d547
   [junit4]   2> 380372 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 380372 INFO  (zkCallback-320-thread-1-processing-n:127.0.0.1:59036_solr) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@fe3fef6 name:ZooKeeperConnection Watcher:127.0.0.1:52735/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 380372 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 380399 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
   [junit4]   2> 380416 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
   [junit4]   2> 380418 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
   [junit4]   2> 380420 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
   [junit4]   2> 380421 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
   [junit4]   2> 380423 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 380432 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 380433 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 380434 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 380435 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 380436 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 380436 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 380437 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 380439 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
   [junit4]   2> 380440 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
   [junit4]   2> 380440 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 380441 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/96591924306247685-127.0.0.1:59036_solr-n_0000000000
   [junit4]   2> 380441 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:59036_solr
   [junit4]   2> 380441 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
   [junit4]   2> 380442 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.Overseer Overseer (id=96591924306247685-127.0.0.1:59036_solr-n_0000000000) starting
   [junit4]   2> 380443 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
   [junit4]   2> 380452 INFO  (zkCallback-321-thread-1-processing-n:127.0.0.1:46719_solr) [n:127.0.0.1:46719_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3c8a1ea4 name:ZooKeeperConnection Watcher:127.0.0.1:52735/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 380452 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 380457 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 380457 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 380458 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 380468 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 380469 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/96591924306247686-127.0.0.1:46719_solr-n_0000000001
   [junit4]   2> 380469 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/96591924306247685-127.0.0.1:59036_solr-n_0000000000 to know if I could be the leader
   [junit4]   2> 380469 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46719_solr
   [junit4]   2> 380469 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:46719_solr
   [junit4]   2> 380491 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 380491 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.CoreContainer Authentication plugin class obtained from system property 'authenticationPlugin': org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 380512 INFO  (zkCallback-320-thread-1-processing-n:127.0.0.1:59036_solr) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 380531 INFO  (zkCallback-320-thread-1-processing-n:127.0.0.1:59036_solr) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 380552 INFO  (zkCallback-321-thread-2-processing-n:127.0.0.1:46719_solr) [n:127.0.0.1:46719_solr    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 380561 INFO  (zkCallback-321-thread-2-processing-n:127.0.0.1:46719_solr) [n:127.0.0.1:46719_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 380565 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 380581 INFO  (OverseerStateUpdate-96591924306247685-127.0.0.1:59036_solr-n_0000000000) [n:127.0.0.1:59036_solr    ] o.a.s.c.Overseer Starting to work on the main queue
   [junit4]   2> 380585 INFO  (OverseerCollectionConfigSetProcessor-96591924306247685-127.0.0.1:59036_solr-n_0000000000) [n:127.0.0.1:59036_solr    ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
   [junit4]   2> 380588 INFO  (OverseerStateUpdate-96591924306247685-127.0.0.1:59036_solr-n_0000000000) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (1)
   [junit4]   2> 380602 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.ZkController Publish node=127.0.0.1:59036_solr as DOWN
   [junit4]   2> 380606 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
   [junit4]   2> 380633 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59036_solr
   [junit4]   2> 380633 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:59036_solr
   [junit4]   2> 380636 INFO  (zkCallback-321-thread-2-processing-n:127.0.0.1:46719_solr) [n:127.0.0.1:46719_solr    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 380637 INFO  (zkCallback-320-thread-1-processing-n:127.0.0.1:59036_solr) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 380637 INFO  (OverseerStateUpdate-96591924306247685-127.0.0.1:59036_solr-n_0000000000) [n:127.0.0.1:59036_solr    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "operation":"downnode",
   [junit4]   2>   "node_name":"127.0.0.1:59036_solr"} current state version: 0
   [junit4]   2> 380637 INFO  (OverseerStateUpdate-96591924306247685-127.0.0.1:59036_solr-n_0000000000) [n:127.0.0.1:59036_solr    ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:59036_solr
   [junit4]   2> 380637 INFO  (zkCallback-321-thread-2-processing-n:127.0.0.1:46719_solr) [n:127.0.0.1:46719_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 380640 INFO  (zkCallback-320-thread-1-processing-n:127.0.0.1:59036_solr) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 380652 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 380668 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.CoreContainer Authentication plugin class obtained from system property 'authenticationPlugin': org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 380668 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
   [junit4]   2> 381134 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 381154 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 381424 INFO  (jetty-launcher-310-thread-1-EventThread) [n:127.0.0.1:59036_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 381428 INFO  (jetty-launcher-310-thread-2-EventThread) [n:127.0.0.1:46719_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 383050 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.CoreContainer Reconfiguring HttpClient settings.
   [junit4]   2> 383051 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_11395C375D5ACE69-001/tempDir-001/node1/.
   [junit4]   2> 383051 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions
   [junit4]   2> 383052 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.s.SolrDispatchFilter user.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0
   [junit4]   2> 383053 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 383057 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.CoreContainer Reconfiguring HttpClient settings.
   [junit4]   2> 383058 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_11395C375D5ACE69-001/tempDir-001/node2/.
   [junit4]   2> 383058 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions
   [junit4]   2> 383066 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.s.SolrDispatchFilter user.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0
   [junit4]   2> 383066 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 383067 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[11395C375D5ACE69]-worker) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 383067 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[11395C375D5ACE69]-worker) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 383075 INFO  (zkCallback-324-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5adbe2a1 name:ZooKeeperConnection Watcher:127.0.0.1:52735 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 383075 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[11395C375D5ACE69]-worker) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 383075 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[11395C375D5ACE69]-worker) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 383172 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[11395C375D5ACE69]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenew
   [junit4]   2> 385786 INFO  (qtp1050296361-2049) [n:127.0.0.1:59036_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVcp1_iXigFXTeR8lwEEFPZYhRvkc3NSscVuzctrvdNO5FZ1B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 385786 INFO  (qtp1050296361-2049) [n:127.0.0.1:59036_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVcp1_iXigFXTeR8lwEEFPZYhRvkc3NSscVuzctrvdNO5FZ1B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 386050 INFO  (qtp1909215633-2062) [n:127.0.0.1:46719_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVcp1_iXigFXTeR8lwEEFPZYhRvkc3NSscVuzctrvdNO5FZ1B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 386050 INFO  (qtp1909215633-2062) [n:127.0.0.1:46719_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVcp1_iXigFXTeR8lwEEFPZYhRvkc3NSscVuzctrvdNO5FZ1B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 386122 INFO  (qtp1050296361-2048) [n:127.0.0.1:59036_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVcp2AJRigFXTeSGUQIEFB0SSck1SVcUjI01BhDg-Ujha20FB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 386122 INFO  (qtp1050296361-2048) [n:127.0.0.1:59036_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVcp2AJRigFXTeSGUQIEFB0SSck1SVcUjI01BhDg-Ujha20FB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 386174 INFO  (qtp1909215633-2066) [n:127.0.0.1:46719_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVcp2AJRigFXTeSGUQIEFB0SSck1SVcUjI01BhDg-Ujha20FB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 386174 INFO  (qtp1909215633-2066) [n:127.0.0.1:46719_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVcp2AJRigFXTeSGUQIEFB0SSck1SVcUjI01BhDg-Ujha20FB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 386242 INFO  (qtp1050296361-2055) [n:127.0.0.1:59036_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVcp2ALaigFXTeSG2gMEFC2kvJld_tDUXzBWKdDxo4jSVwwGB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 386242 INFO  (qtp1050296361-2055) [n:127.0.0.1:59036_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVcp2ALaigFXTeSG2gMEFC2kvJld_tDUXzBWKdDxo4jSVwwGB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 386274 INFO  (qtp1909215633-2066) [n:127.0.0.1:46719_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVcp2ALaigFXTeSG2gMEFC2kvJld_tDUXzBWKdDxo4jSVwwGB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 386274 INFO  (qtp1909215633-2066) [n:127.0.0.1:46719_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVcp2ALaigFXTeSG2gMEFC2kvJld_tDUXzBWKdDxo4jSVwwGB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 386322 INFO  (qtp1050296361-2055) [n:127.0.0.1:59036_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVcp2AMsigFXTeSHLAQEFJkoiydYtc0rglcHAQRxYTGuLPpGB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 386322 INFO  (qtp1050296361-2055) [n:127.0.0.1:59036_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVcp2AMsigFXTeSHLAQEFJkoiydYtc0rglcHAQRxYTGuLPpGB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 386338 INFO  (qtp1909215633-2066) [n:127.0.0.1:46719_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVcp2AMsigFXTeSHLAQEFJkoiydYtc0rglcHAQRxYTGuLPpGB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 386338 INFO  (qtp1909215633-2066) [n:127.0.0.1:46719_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVcp2AMsigFXTeSHLAQEFJkoiydYtc0rglcHAQRxYTGuLPpGB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 386354 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[11395C375D5ACE69]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenew
   [junit4]   2> 386379 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[11395C375D5ACE69]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenewFail
   [junit4]   2> 386389 WARN  (qtp1050296361-2048) [n:127.0.0.1:59036_solr    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
   [junit4]   2> 386390 WARN  (qtp1050296361-2048) [n:127.0.0.1:59036_solr    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.kerberosplugin.requestcontinues
   [junit4]   2> 386403 WARN  (qtp1909215633-2063) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
   [junit4]   2> 386425 WARN  (qtp1909215633-2063) [    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.kerberosplugin.requestcontinues
   [junit4]   2> 386435 WARN  (qtp1050296361-2052) [    ] o.a.h.s.a.s.Authentication

[...truncated too long message...]

b/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/bcprov-jdk15-1.45.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/cglib-nodep-2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/commons-math3-3.4.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/easymock-3.0.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-common-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-hdfs-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-minikdc-2.7.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-annotations-2.5.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-core-asl-1.9.13.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-databind-2.5.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-mapper-asl-1.9.13.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/mockito-core-1.9.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/objenesis-1.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/presto-parser-0.122.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/slice-0.10.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/analysis/icu/lucene-analyzers-icu-7.0.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/analysis-extras/lib/icu4j-56.1.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_102/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.3.4.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20160914_175010_876.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20160914_175010_876.suites -stdin
   [junit4] ERROR: JVM J2 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1501)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:121)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:955)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:952)
   [junit4] 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4] 	at java.lang.Thread.run(Thread.java:745)

BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:707: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build.xml:233: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/common-build.xml:536: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:1443: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:984: At least one slave process threw an exception, first: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.

Total time: 391 minutes 23 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
ERROR: Failed to archive {checkout/solr/build/solr-core/test/temp/junit4-J2-20160914_175010_876.events=checkout/solr/build/solr-core/test/temp/junit4-J2-20160914_175010_876.events, checkout/solr/build/solr-core/test/temp/junit4-J0-20160914_175010_876.events=checkout/solr/build/solr-core/test/temp/junit4-J0-20160914_175010_876.events, checkout/README.txt=checkout/README.txt, checkout/solr/build/solr-core/test/temp/junit4-J1-20160914_175010_883.events=checkout/solr/build/solr-core/test/temp/junit4-J1-20160914_175010_883.events} due to internal error; falling back to full archiving
java.lang.NullPointerException
	at jsync.protocol.ChangeInputStream.read(ChangeInputStream.java:69)
	at com.cloudbees.jenkins.plugins.jsync.archiver.MD5DigestingInputStream.read(MD5DigestingInputStream.java:39)
	at com.google.common.io.LimitInputStream.read(LimitInputStream.java:79)
	at java.io.FilterInputStream.read(FilterInputStream.java:107)
	at com.google.common.io.ByteStreams.copy(ByteStreams.java:193)
	at jsync.protocol.FileSequenceReader.read(FileSequenceReader.java:35)
	at com.cloudbees.jenkins.plugins.jsync.archiver.JSyncArtifactManager.remoteSync(JSyncArtifactManager.java:158)
	at com.cloudbees.jenkins.plugins.jsync.archiver.JSyncArtifactManager.archive(JSyncArtifactManager.java:76)
	at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:238)
	at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:78)
	at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
	at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:779)
	at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:720)
	at hudson.model.Build$BuildExecution.post2(Build.java:185)
	at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:665)
	at hudson.model.Run.execute(Run.java:1766)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:98)
	at hudson.model.Executor.run(Executor.java:410)
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1115 - Failure

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1115/

4 tests failed.
FAILED:  org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:38503/ff_/i

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:38503/ff_/i
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:477)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  org.apache.solr.handler.component.DistributedFacetPivotLongTailTest.test

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded
	at __randomizedtesting.SeedInfo.seed([B9E25108DCDC63C8:31B66ED272200E30]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:347)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:152)
	at org.apache.solr.handler.component.DistributedFacetPivotLongTailTest.test(DistributedFacetPivotLongTailTest.java:71)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
	at __randomizedtesting.SeedInfo.seed([B9E25108DCDC63C8]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.GeneratedMethodAccessor99.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest:     1) Thread[id=197403, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=197393, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=197423, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=197408, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=197418, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=197398, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=197378, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=197383, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=197413, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=197388, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 
   1) Thread[id=197403, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=197393, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=197423, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=197408, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=197418, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=197398, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=197378, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=197383, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=197413, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=197388, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([B9E25108DCDC63C8]:0)




Build Log:
[...truncated 12778 lines...]
   [junit4] Suite: org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_B9E25108DCDC63C8-001/init-core-data-001
   [junit4]   2> 5777874 INFO  (SUITE-ConcurrentDeleteAndCreateCollectionTest-seed#[B9E25108DCDC63C8]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 5777877 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testConcurrentCreateAndDeleteDoesNotFail
   [junit4]   2> 5777877 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 5777886 INFO  (Thread-181698) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 5777887 INFO  (Thread-181698) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 5778051 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.ZkTestServer start zk server on port:43889
   [junit4]   2> 5778051 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778053 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778073 INFO  (zkCallback-3735-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@feb53a6 name:ZooKeeperConnection Watcher:127.0.0.1:43889 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778073 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778073 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778074 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr/solr.xml
   [junit4]   2> 5778079 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr/clusterprops.json
   [junit4]   2> 5778086 INFO  (jetty-launcher-3734-thread-1) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 5778093 INFO  (jetty-launcher-3734-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@75838d63{/solr,null,AVAILABLE}
   [junit4]   2> 5778095 INFO  (jetty-launcher-3734-thread-1) [    ] o.e.j.s.ServerConnector Started ServerConnector@6f98c6eb{SSL,[ssl, http/1.1]}{127.0.0.1:60459}
   [junit4]   2> 5778095 INFO  (jetty-launcher-3734-thread-1) [    ] o.e.j.s.Server Started @5785706ms
   [junit4]   2> 5778099 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=60459}
   [junit4]   2> 5778099 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 5778099 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_B9E25108DCDC63C8-001/tempDir-001/node1'
   [junit4]   2> 5778099 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 5778100 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 5778100 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778110 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778110 INFO  (zkCallback-3736-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@753ea486 name:ZooKeeperConnection Watcher:127.0.0.1:43889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778110 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778110 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778110 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 5778123 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_B9E25108DCDC63C8-001/tempDir-001/node1/.
   [junit4]   2> 5778123 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.CoreContainer New CoreContainer 1450366150
   [junit4]   2> 5778123 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_B9E25108DCDC63C8-001/tempDir-001/node1]
   [junit4]   2> 5778123 WARN  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.CoreContainer Couldn't add files from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_B9E25108DCDC63C8-001/tempDir-001/node1/lib to classpath: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_B9E25108DCDC63C8-001/tempDir-001/node1/lib
   [junit4]   2> 5778124 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : https,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,
   [junit4]   2> 5778126 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.u.UpdateShardHandler Created UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&maxConnections=100000&maxConnectionsPerHost=100000
   [junit4]   2> 5778126 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 5778126 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 5778126 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43889/solr
   [junit4]   2> 5778126 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 5778126 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778127 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778143 INFO  (zkCallback-3739-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@13d9a076 name:ZooKeeperConnection Watcher:127.0.0.1:43889 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778143 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778143 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778146 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$12/1518713987@4a0f334
   [junit4]   2> 5778155 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778158 INFO  (zkCallback-3740-thread-1-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@60ab6f6d name:ZooKeeperConnection Watcher:127.0.0.1:43889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778158 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778159 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
   [junit4]   2> 5778160 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
   [junit4]   2> 5778162 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
   [junit4]   2> 5778164 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
   [junit4]   2> 5778172 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
   [junit4]   2> 5778173 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 5778174 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 5778174 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 5778175 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 5778179 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 5778184 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 5778185 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ZkStateReader Loaded cluster properties: {urlScheme=https}
   [junit4]   2> 5778185 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 5778187 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
   [junit4]   2> 5778197 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
   [junit4]   2> 5778198 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 5778198 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/96581753093685251-127.0.0.1:60459_solr-n_0000000000
   [junit4]   2> 5778198 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:60459_solr
   [junit4]   2> 5778198 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
   [junit4]   2> 5778208 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.Overseer Overseer (id=96581753093685251-127.0.0.1:60459_solr-n_0000000000) starting
   [junit4]   2> 5778209 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
   [junit4]   2> 5778222 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 5778223 INFO  (OverseerCollectionConfigSetProcessor-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
   [junit4]   2> 5778231 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60459_solr
   [junit4]   2> 5778231 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:60459_solr
   [junit4]   2> 5778231 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.Overseer Starting to work on the main queue
   [junit4]   2> 5778247 INFO  (zkCallback-3740-thread-1-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 5778248 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 5778275 INFO  (zkCallback-3740-thread-1-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (1)
   [junit4]   2> 5778280 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 5778280 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 5778281 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_B9E25108DCDC63C8-001/tempDir-001/node1/.
   [junit4]   2> 5778281 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions
   [junit4]   2> 5778295 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.s.SolrDispatchFilter user.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2
   [junit4]   2> 5778295 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 5778299 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778301 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778301 INFO  (zkCallback-3742-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4f6830a0 name:ZooKeeperConnection Watcher:127.0.0.1:43889 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778301 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778301 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778316 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778347 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778349 INFO  (zkCallback-3744-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5feb1f6e name:ZooKeeperConnection Watcher:127.0.0.1:43889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778349 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778350 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778350 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection0/schema.xml
   [junit4]   2> 5778352 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection0/solrconfig.xml
   [junit4]   2> 5778367 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778376 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778377 INFO  (zkCallback-3745-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@60b9fb04 name:ZooKeeperConnection Watcher:127.0.0.1:43889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778377 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778377 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778378 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection1/schema.xml
   [junit4]   2> 5778387 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection1/solrconfig.xml
   [junit4]   2> 5778399 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778400 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778410 INFO  (zkCallback-3746-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@24ba16e2 name:ZooKeeperConnection Watcher:127.0.0.1:43889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778410 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778410 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778411 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection2/schema.xml
   [junit4]   2> 5778463 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection2/solrconfig.xml
   [junit4]   2> 5778474 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778479 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778499 INFO  (zkCallback-3747-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@138f6e63 name:ZooKeeperConnection Watcher:127.0.0.1:43889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778499 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778499 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778500 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection3/schema.xml
   [junit4]   2> 5778505 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection3/solrconfig.xml
   [junit4]   2> 5778518 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778523 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778527 INFO  (zkCallback-3748-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@7cb113fc name:ZooKeeperConnection Watcher:127.0.0.1:43889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778527 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778528 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778528 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection4/schema.xml
   [junit4]   2> 5778530 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection4/solrconfig.xml
   [junit4]   2> 5778537 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778547 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778549 INFO  (zkCallback-3749-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@42690972 name:ZooKeeperConnection Watcher:127.0.0.1:43889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778549 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778549 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778550 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection5/schema.xml
   [junit4]   2> 5778557 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection5/solrconfig.xml
   [junit4]   2> 5778600 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778603 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778615 INFO  (zkCallback-3750-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5935097b name:ZooKeeperConnection Watcher:127.0.0.1:43889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778615 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778616 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778616 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection6/schema.xml
   [junit4]   2> 5778659 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection6/solrconfig.xml
   [junit4]   2> 5778687 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778701 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778715 INFO  (zkCallback-3751-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5e246904 name:ZooKeeperConnection Watcher:127.0.0.1:43889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778715 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778715 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778716 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection7/schema.xml
   [junit4]   2> 5778717 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection7/solrconfig.xml
   [junit4]   2> 5778765 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778795 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778796 INFO  (zkCallback-3752-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@64985ce9 name:ZooKeeperConnection Watcher:127.0.0.1:43889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778796 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778796 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778797 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection8/schema.xml
   [junit4]   2> 5778798 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection8/solrconfig.xml
   [junit4]   2> 5778807 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778827 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778835 INFO  (zkCallback-3753-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3ff66111 name:ZooKeeperConnection Watcher:127.0.0.1:43889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778835 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778835 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778836 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection9/schema.xml
   [junit4]   2> 5778838 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection9/solrconfig.xml
   [junit4]   2> 5779276 INFO  (qtp1703007059-197353) [n:127.0.0.1:60459_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=collection0&name=collection0&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 5779278 INFO  (OverseerCollectionConfigSetProcessor-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2>   "name":"collection0",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection0",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779281 INFO  (qtp1703007059-197346) [n:127.0.0.1:60459_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=collection1&name=collection1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 5779329 INFO  (qtp1703007059-197347) [n:127.0.0.1:60459_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=collection4&name=collection4&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 5779340 INFO  (qtp1703007059-197352) [n:127.0.0.1:60459_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=collection8&name=collection8&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 5779343 INFO  (qtp1703007059-197428) [n:127.0.0.1:60459_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=collection9&name=collection9&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 5779345 INFO  (qtp1703007059-197351) [n:127.0.0.1:60459_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=collection7&name=collection7&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 5779347 INFO  (OverseerCollectionConfigSetProcessor-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000006 message:{
   [junit4]   2>   "name":"collection7",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection7",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779348 INFO  (OverseerThreadFactory-8116-thread-1-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"collection0",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection0",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779348 INFO  (OverseerThreadFactory-8116-thread-1-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection0
   [junit4]   2> 5779352 INFO  (qtp1703007059-197349) [n:127.0.0.1:60459_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=collection3&name=collection3&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 5779391 INFO  (OverseerThreadFactory-8116-thread-2-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"collection7",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection7",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779391 INFO  (OverseerThreadFactory-8116-thread-2-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection7
   [junit4]   2> 5779411 INFO  (qtp1703007059-197427) [n:127.0.0.1:60459_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=collection2&name=collection2&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 5779412 INFO  (qtp1703007059-197431) [n:127.0.0.1:60459_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=collection5&name=collection5&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 5779428 INFO  (OverseerThreadFactory-8116-thread-1-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection0 
   [junit4]   2> 5779428 INFO  (OverseerThreadFactory-8116-thread-2-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection7 
   [junit4]   2> 5779430 INFO  (OverseerThreadFactory-8116-thread-1-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection0
   [junit4]   2> 5779431 INFO  (zkCallback-3740-thread-1-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 5779431 INFO  (OverseerCollectionConfigSetProcessor-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000003 message:{
   [junit4]   2>   "name":"collection4",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection4",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779434 INFO  (OverseerThreadFactory-8116-thread-2-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection7
   [junit4]   2> 5779448 INFO  (qtp1703007059-197350) [n:127.0.0.1:60459_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=collection6&name=collection6&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 5779547 INFO  (OverseerCollectionConfigSetProcessor-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000004 message:{
   [junit4]   2>   "name":"collection8",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection8",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779547 INFO  (OverseerThreadFactory-8116-thread-3-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"collection4",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection4",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779547 INFO  (OverseerCollectionConfigSetProcessor-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000005 message:{
   [junit4]   2>   "name":"collection9",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection9",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779548 INFO  (OverseerThreadFactory-8116-thread-3-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection4
   [junit4]   2> 5779548 INFO  (OverseerThreadFactory-8116-thread-4-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"collection8",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection8",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779548 INFO  (OverseerThreadFactory-8116-thread-4-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection8
   [junit4]   2> 5779549 INFO  (zkCallback-3740-thread-2-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 5779549 INFO  (OverseerThreadFactory-8116-thread-5-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"collection9",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection9",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779549 INFO  (OverseerThreadFactory-8116-thread-5-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection9
   [junit4]   2> 5779552 INFO  (OverseerThreadFactory-8116-thread-3-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection4 
   [junit4]   2> 5779552 INFO  (OverseerThreadFactory-8116-thread-4-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection8 
   [junit4]   2> 5779553 INFO  (OverseerThreadFactory-8116-thread-5-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection9 
   [junit4]   2> 5779567 INFO  (OverseerThreadFactory-8116-thread-4-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection8
   [junit4]   2> 5779567 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
   [junit4]   2>   "name":"collection0",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection0",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 5779567 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection0
   [junit4]   2> 5779567 INFO  (OverseerThreadFactory-8116-thread-3-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection4
   [junit4]   2> 5779567 INFO  (OverseerThreadFactory-8116-thread-5-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection9
   [junit4]   2> 5779573 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection0/state.json
   [junit4]   2> 5779574 INFO  (zkCallback-3740-thread-1-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 5779587 INFO  (OverseerCollectionConfigSetProcessor-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000002 message:{
   [junit4]   2>   "name":"collection1",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection1",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779590 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.Overseer processMessage: queueSize: 3, message = {
   [junit4]   2>   "name":"collection7",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection7",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 5779590 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection7
   [junit4]   2> 5779598 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
   [junit4]   2>   "name":"collection4",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection4",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 5779598 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection4
   [junit4]   2> 5779604 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection7/state.json
   [junit4]   2> 5779606 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "name":"collection8",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection8",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 5779606 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection8
   [junit4]   2> 5779607 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection4/state.json
   [junit4]   2> 5779612 INFO  (OverseerCollectionConfigSetProcessor-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000010 message:{
   [junit4]   2>   "name":"collection3",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection3",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779614 INFO  (OverseerThreadFactory-8116-thread-6-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"collection1",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection1",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779615 INFO  (OverseerThreadFactory-8116-thread-6-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 5779627 INFO  (OverseerThreadFactory-8116-thread-6-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection1 
   [junit4]   2> 5779628 INFO  (OverseerThreadFactory-8116-thread-6-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1
   [junit4]   2> 5779628 INFO  (zkCallback-3740-thread-2-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 5779629 INFO  (zkCallback-3740-thread-2-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 5779636 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "name":"collection1",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection1",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 5779636 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection1
   [junit4]   2> 5779636 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection8/state.json
   [junit4]   2> 5779640 INFO  (OverseerThreadFactory-8116-thread-1-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Creating SolrCores for new collection collection0, shardNames [shard1] , replicationFactor : 1
   [junit4]   2> 5779640 INFO  (OverseerThreadFactory-8116-thread-1-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Creating core collection0_shard1_replica1 as part of shard shard1 of collection collection0 on 127.0.0.1:60459_solr
   [junit4]   2> 5779643 INFO  (OverseerThreadFactory-8116-thread-2-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Creating SolrCores for new collection collection7, shardNames [shard1] , replicationFactor : 1
   [junit4]   2> 5779643 INFO  (OverseerThreadFactory-8116-thread-2-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Creating core collection7_shard1_replica1 as part of shard shard1 of collection collection7 on 127.0.0.1:60459_solr
   [junit4]   2> 5779667 INFO  (OverseerCollectionConfigSetProcessor-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000011 message:{
   [junit4]   2>   "name":"collection2",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection2",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779667 INFO  (OverseerThreadFactory-8116-thread-7-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"collection3",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection3",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779667 INFO  (OverseerThreadFactory-8116-thread-7-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection3
   [junit4]   2> 5779668 INFO  (OverseerThreadFactory-8116-thread-7-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection3 
   [junit4]   2> 5779668 INFO  (OverseerThreadFactory-8116-thread-7-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection3
   [junit4]   2> 5779670 INFO  (OverseerCollectionConfigSetProcessor-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000012 message:{
   [junit4]   2>   "name":"collection5",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection5",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779670 INFO  (OverseerThreadFactory-8116-thread-8-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"collection2",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection2",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779670 INFO  (OverseerThreadFactory-8116-thread-8-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection2
   [junit4]   2> 5779673 INFO  (OverseerThreadFactory-8116-thread-8-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection2 
   [junit4]   2> 5779673 INFO  (OverseerThreadFactory-8116-thread-8-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection2
   [junit4]   2> 5779676 INFO  (OverseerCollectionConfigSetProcessor-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000018 message:{
   [junit4]   2>   "name":"collection6",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection6",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779676 INFO  (OverseerThreadFactory-8116-thread-9-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"collection5",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection5",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779676 INFO  (OverseerThreadFactory-8116-thread-9-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection5
   [junit4]   2> 5779686 INFO  (OverseerThreadFactory-8116-thread-9-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection5 
   [junit4]   2> 5779687 INFO  (OverseerThreadFactory-8116-thread-9-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection5
   [junit4]   2> 5779695 INFO  (OverseerThreadFactory-8116-thread-10-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"collection6",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection6",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779695 INFO  (OverseerThreadFactory-8116-thread-10-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection6
   [junit4]   2> 5779699 INFO  (OverseerThreadFactory-8116-thread-3-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Creating SolrCores for new collection collection4, shardNames [shard1] , replicationFactor : 1
   [junit4]   2> 5779699 INFO  (OverseerThreadFactory-8116-thread-3-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Creating core collection4_shard1_replica1 as part of shard shard1 of collection collection4 on 127.0.0.1:60459_solr
   [junit4]   2> 5779699 INFO  (OverseerThreadFactory-8116-thread-4-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Creating SolrCores for new collection collection8, shardNames [shard1] , replicationFactor : 1
   [junit4]   2> 5779699 INFO  (OverseerThreadFactory-8116-thread-4-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Creating core collection8_shard1_replica1 as part of shard shard1 of collection collection8 on 127.0.0.1:60459_solr
   [junit4]   2> 5779710 INFO  (OverseerThreadFactory-8116-thread-10-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection6 
   [junit4]   2> 5779710 INFO  (OverseerThreadFactory-8116-thread-10-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection6
   [junit4]   2> 5779739 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection1/state.json
   [junit4]   2> 5779741 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.Overseer processMessage: queueSize: 5, message = {
   [junit4]   2>   "name":"collection9",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection9",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 5779741 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection9
   [junit4]   2> 5779743 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.Overseer processMessage: queueSize: 0, message = {
   [junit4]   2>   "name":"collection3",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection3",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 5779743 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection3
   [junit4]   2> 5779751 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection9/state.json
   [junit4]   2> 5779754 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.Overseer processMessage: queueSize: 0, message = {
   [junit4]   2>   "name":"collection2",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection2",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 5779754 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection2
   [junit4]   2> 5779754 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection3/state.json
   [junit4]   2> 5779756 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.Overseer processMessage: queueSize: 0, message = {
   [junit4]   2>   "name":"collection5",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection5",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 5779756 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection5
   [junit4]   2> 5779757 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection2/state.json
   [junit4]   2> 5779765 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.Overseer processMessage: queueSize: 0, message = {
   [junit4]   2>   "name":"collection6",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection6",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 5779765 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection6
   [junit4]   2> 5779765 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection5/state.json
   [junit4]   2> 5779838 INFO  (OverseerThreadFactory-8116-thread-6-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Creating SolrCores for new collection collection1, shardNames [shard1] , replicationFactor : 1
   [junit4]   2> 5779838 INFO  (OverseerThreadFactory-8116-thread-6-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Creating core collection1_shard1_replica1 as part of shard shard1 of collection collection1 on 127.0.0.1:60459_solr
   [junit4]   2> 5779856 INFO  (OverseerThreadFactory-8116-thread-5-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Creating SolrCores for new collection collection9, shardNames [shard1] , replicationFactor : 1
   [junit4]   2> 5779856 INFO  (OverseerThreadFactory-8116-thread-5-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Creating core collection9_shard1_replica1 as part of shard shard1 of collection collection9 on 127.0.0.1:60459_solr
   [junit4]   2> 5779871 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection6/state.json
   [junit4]   2> 5780157 INFO  (zkCallback-3740-thread-1-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 5780205 INFO  (qtp1703007059-197433) [n:127.0.0.1:60459_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=collection0&name=collection0_shard1_replica1&action=CREATE&numShards=1&collection=collection0&shard=shard1&wt=javabin&version=2
   [junit4]   2> 5780206 INFO  (qtp1703007059-197435) [n:127.0.0.1:60459_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=collection9&name=collection9_shard1_replica1&action=CREATE&numShards=1&collection=collection9&shard=shard1&wt=javabin&version=2
   [junit4]   2> 5780206 INFO  (qtp1703007059-197433) [n:127.0.0.1:60459_solr    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection0_shard1_replica1, collection.configName=collection0, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.json, collection=collection0, numShards=1, dataDir=data/, shard=shard1}
   [junit4]   2> 5780206 INFO  (qtp1703007059-197435) [n:127.0.0.1:60459_solr    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection9_shard1_replica1, collection.configName=collection9, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.json, collection=collection9, numShards=1, dataDir=data/, shard=shard1}
   [junit4]   2> 5780206 INFO  (qtp1703007059-197426) [n:127.0.0.1:60459_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=collection7&name=collection7_shard1_replica1&action=CREATE&numShards=1&collection=collection7&shard=shard1&wt=javabin&version=2
   [junit4]   2> 5780206 INFO  (qtp1703007059-197426) [n:127.0.0.1:60459_solr    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection7_shard1_replica1, collection.configName=collection7, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.json, collection=collection7, numShards=1, dataDir=data/, shard=shard1}
   [junit4]   2> 5780207 INFO  (qtp1703007059-197426) [n:127.0.0.1:60459_solr c:collection7 s:shard1  x:collection7_shard1_replica1] o.a.s.c.ZkController publishing state=down
   [junit4]   2> 5780208 INFO  (qtp1703007059-197426) [n:127.0.0.1:60459_solr c:collection7 s:shard1  x:collection7_shard1_replica1] o.a.s.c.ZkController Registering watch for collection collection7
   [junit4]  

[...truncated too long message...]

ection1/data
   [junit4]   2> 7480269 INFO  (coreCloseExecutor-11532-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory looking to close /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedFacetPivotLongTailTest_B9E25108DCDC63C8-001/tempDir-001/shard2/cores/collection1/data/index [CachedDir<<refCount=0;path=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedFacetPivotLongTailTest_B9E25108DCDC63C8-001/tempDir-001/shard2/cores/collection1/data/index;done=false>>]
   [junit4]   2> 7480269 INFO  (coreCloseExecutor-11532-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory Closing directory, CoreContainer#isShutdown=true
   [junit4]   2> 7480269 INFO  (coreCloseExecutor-11532-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory Closing directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedFacetPivotLongTailTest_B9E25108DCDC63C8-001/tempDir-001/shard2/cores/collection1/data/index
   [junit4]   2> 7480271 INFO  (TEST-DistributedFacetPivotLongTailTest.test-seed#[B9E25108DCDC63C8]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@55954cfb{/,null,UNAVAILABLE}
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=DistributedFacetPivotLongTailTest -Dtests.method=test -Dtests.seed=B9E25108DCDC63C8 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=is -Dtests.timezone=Mexico/BajaSur -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR    116s J0 | DistributedFacetPivotLongTailTest.test <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([B9E25108DCDC63C8:31B66ED272200E30]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:347)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:152)
   [junit4]    > 	at org.apache.solr.handler.component.DistributedFacetPivotLongTailTest.test(DistributedFacetPivotLongTailTest.java:71)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
   [junit4]   2> 7480332 INFO  (SUITE-DistributedFacetPivotLongTailTest-seed#[B9E25108DCDC63C8]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedFacetPivotLongTailTest_B9E25108DCDC63C8-001
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=ClassicSimilarity, locale=is, timezone=Mexico/BajaSur
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=92830728,total=477626368
   [junit4]   2> NOTE: All tests run in this JVM: [TestRealTimeGet, TestFieldCacheVsDocValues, ExplicitHLLTest, PreAnalyzedUpdateProcessorTest, TestCloudPseudoReturnFields, TestScoreJoinQPNoScore, MBeansHandlerTest, SolrCloudExampleTest, ZkNodePropsTest, TestNoOpRegenerator, TestFoldingMultitermQuery, TestUseDocValuesAsStored, TestJsonFacets, TestRandomDVFaceting, TestCustomSort, ResponseHeaderTest, TestIBSimilarityFactory, HdfsNNFailoverTest, UUIDFieldTest, TestRawTransformer, TestConfigSetProperties, TestCopyFieldCollectionResource, RequiredFieldsTest, TestPKIAuthenticationPlugin, TestHdfsUpdateLog, ReturnFieldsTest, RegexBoostProcessorTest, TestStressRecovery, AlternateDirectoryTest, TestNumericTerms32, TestPseudoReturnFields, BlockJoinFacetRandomTest, TestFaceting, WrapperMergePolicyFactoryTest, LeaderInitiatedRecoveryOnShardRestartTest, FileBasedSpellCheckerTest, TestComponentsName, RulesTest, HdfsChaosMonkeyNothingIsSafeTest, TestLocalFSCloudBackupRestore, DateRangeFieldTest, OverseerTaskQueueTest, ScriptEngineTest, TestNRTOpen, TestReplicationHandler, NumericFieldsTest, TestTrieFacet, DistributedSuggestComponentTest, TestRecoveryHdfs, TestJoin, TestRandomFlRTGCloud, TestSolr4Spatial, SynonymTokenizerTest, TestExpandComponent, TestMacroExpander, TestSolrCloudWithDelegationTokens, TestRandomCollapseQParserPlugin, ChaosMonkeyNothingIsSafeTest, CollectionStateFormat2Test, StandardRequestHandlerTest, CleanupOldIndexTest, TestHighlightDedupGrouping, CursorPagingTest, TestValueSourceCache, TestGeoJSONResponseWriter, TestTolerantUpdateProcessorRandomCloud, ConnectionReuseTest, DocumentAnalysisRequestHandlerTest, HLLUtilTest, TestDynamicFieldResource, DataDrivenBlockJoinTest, TestBackupRepositoryFactory, TestRestManager, QueryEqualityTest, DistributedFacetPivotWhiteBoxTest, BasicAuthIntegrationTest, CoreAdminRequestStatusTest, PeerSyncTest, ResourceLoaderTest, TestHdfsCloudBackupRestore, TestFastWriter, OutOfBoxZkACLAndCredentialsProvidersTest, NotRequiredUniqueKeyTest, TestReqParamsAPI, TestPHPSerializedResponseWriter, TestSchemaResource, TestReplicaProperties, TestHighFrequencyDictionaryFactory, AnalysisAfterCoreReloadTest, TestFunctionQuery, XsltUpdateRequestHandlerTest, ClusterStateTest, DefaultValueUpdateProcessorTest, TestStressCloudBlindAtomicUpdates, TestSolrConfigHandler, UUIDUpdateProcessorFallbackTest, AsyncCallRequestStatusResponseTest, CdcrRequestHandlerTest, ZkStateWriterTest, TestSolrDeletionPolicy2, TestDistribDocBasedVersion, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest, FullSolrCloudDistribCmdsTest, ZkSolrClientTest, TestDistributedGrouping, TestHashPartitioner, ZkControllerTest, TestStressReorder, HardAutoCommitTest, TestRangeQuery, SolrCoreTest, SpellCheckComponentTest, TestGroupingSearch, HighlighterTest, ShowFileRequestHandlerTest, CurrencyFieldOpenExchangeTest, DistributedQueryElevationComponentTest, SimplePostToolTest, TestCoreDiscovery, CoreAdminHandlerTest, TestStressLucene, SuggesterWFSTTest, PolyFieldTest, TestAtomicUpdateErrorCases, SortByFunctionTest, DocValuesMultiTest, DebugComponentTest, IndexBasedSpellCheckerTest, DisMaxRequestHandlerTest, TestWriterPerf, XmlUpdateRequestHandlerTest, PathHierarchyTokenizerFactoryTest, TestIndexingPerformance, MoreLikeThisHandlerTest, FastVectorHighlighterTest, TestJmxIntegration, UniqFieldsUpdateProcessorFactoryTest, TestPartialUpdateDeduplication, SearchHandlerTest, ExternalFileFieldSortTest, TestSolrCoreProperties, SpellPossibilityIteratorTest, TestCodecSupport, TestBM25SimilarityFactory, TestFastOutputStream, TestFastLRUCache, TestSystemIdResolver, DateFieldTest, SpellingQueryConverterTest, DOMUtilTest, SliceStateTest, SystemInfoHandlerTest, CircularListTest, DistributedIntervalFacetingTest, CdcrVersionReplicationTest, CollectionTooManyReplicasTest, CollectionsAPISolrJTest, ConfigSetsAPITest, ConnectionManagerTest, CustomCollectionTest, DistribDocExpirationUpdateProcessorTest, HttpPartitionTest, SaslZkACLProviderTest, TestConfigSetsAPIExclusivity, TestDownShardTolerantSearch, TestLeaderElectionZkExpiry, TestLeaderInitiatedRecoveryThread, TestSizeLimitedDistributedMap, VMParamsZkACLAndCredentialsProvidersTest, HdfsChaosMonkeySafeLeaderTest, HdfsCollectionsAPIDistributedZkTest, HdfsUnloadDistributedZkTest, HdfsWriteToMultipleCollectionsTest, StressHdfsTest, CachingDirectoryFactoryTest, TestImplicitCoreProperties, TestReloadAndDeleteDocs, TestShardHandlerFactory, TestSolrDynamicMBean, TestSolrXml, TestSolrCoreSnapshots, RequestLoggingTest, TestBlobHandler, TestReplicationHandlerBackup, TestSolrConfigHandlerConcurrent, DistributedFacetExistsSmallTest, DistributedFacetPivotLongTailTest]
   [junit4] Completed [554/631 (3!)] on J0 in 116.56s, 1 test, 1 error <<< FAILURES!

[...truncated 222 lines...]
   [junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20160912_211331_976.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid12824.hprof ...
   [junit4] Heap dump file created [494369350 bytes in 7.364 secs]
   [junit4] <<< JVM J0: EOF ----

[...truncated 11045 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:715: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid12824.hprof

Total time: 446 minutes 3 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1114 - Still unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1114/

9 tests failed.
FAILED:  org.apache.lucene.analysis.core.TestRandomChains.testRandomChainsWithLargeStrings

Error Message:


Stack Trace:
java.util.NoSuchElementException
	at __randomizedtesting.SeedInfo.seed([B9ED0ECBBD7CADDD:D3B6B1DAE4328D2E]:0)
	at java.util.TreeMap.key(TreeMap.java:1327)
	at java.util.TreeMap.lastKey(TreeMap.java:297)
	at java.util.TreeSet.last(TreeSet.java:401)
	at org.apache.lucene.analysis.minhash.MinHashFilter$FixedSizeTreeSet.add(MinHashFilter.java:325)
	at org.apache.lucene.analysis.minhash.MinHashFilter.incrementToken(MinHashFilter.java:159)
	at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:67)
	at org.apache.lucene.analysis.util.ElisionFilter.incrementToken(ElisionFilter.java:52)
	at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:67)
	at org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter.incrementToken(ASCIIFoldingFilter.java:102)
	at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:67)
	at org.apache.lucene.analysis.FilteringTokenFilter.incrementToken(FilteringTokenFilter.java:51)
	at org.apache.lucene.analysis.ngram.NGramTokenFilter.incrementToken(NGramTokenFilter.java:98)
	at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:67)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkResetException(BaseTokenStreamTestCase.java:405)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:510)
	at org.apache.lucene.analysis.core.TestRandomChains.testRandomChainsWithLargeStrings(TestRandomChains.java:959)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationStartStop

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard2

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard2
	at __randomizedtesting.SeedInfo.seed([69C7C3B4D964326A:EA041D27CDBCB9E3]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationStartStop(CdcrReplicationDistributedZkTest.java:168)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.core.HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig

Error Message:
The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~134.217728 MB.

Stack Trace:
java.lang.RuntimeException: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~134.217728 MB.
	at __randomizedtesting.SeedInfo.seed([69C7C3B4D964326A:9E680A9F24EDD841]:0)
	at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:304)
	at org.apache.solr.core.HdfsDirectoryFactory.getBlockDirectoryCache(HdfsDirectoryFactory.java:286)
	at org.apache.solr.core.HdfsDirectoryFactory.create(HdfsDirectoryFactory.java:220)
	at org.apache.solr.core.HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig(HdfsDirectoryFactoryTest.java:108)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.OutOfMemoryError: Direct buffer memory
	at java.nio.Bits.reserveMemory(Bits.java:693)
	at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123)
	at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)
	at org.apache.solr.store.blockcache.BlockCache.<init>(BlockCache.java:63)
	at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:302)
	... 42 more


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard2

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard2
	at __randomizedtesting.SeedInfo.seed([69C7C3B4D964326A:35DA24351744865C]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart(CdcrReplicationDistributedZkTest.java:236)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.handler.TestSolrConfigHandlerCloud

Error Message:
ObjectTracker found 8 object(s) that were not released!!! [MDCAwareThreadPoolExecutor, MockDirectoryWrapper, SolrCore, MockDirectoryWrapper, MDCAwareThreadPoolExecutor, SolrCore, MDCAwareThreadPoolExecutor, MockDirectoryWrapper]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 8 object(s) that were not released!!! [MDCAwareThreadPoolExecutor, MockDirectoryWrapper, SolrCore, MockDirectoryWrapper, MDCAwareThreadPoolExecutor, SolrCore, MDCAwareThreadPoolExecutor, MockDirectoryWrapper]
	at __randomizedtesting.SeedInfo.seed([69C7C3B4D964326A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=17331, name=testExecutor-3503-thread-6, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=17331, name=testExecutor-3503-thread-6, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:49991: Error CREATEing SolrCore 'multiunload5': Unable to create core [multiunload5] Caused by: Direct buffer memory
	at __randomizedtesting.SeedInfo.seed([69C7C3B4D964326A]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:608)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.BasicDistributedZkTest.lambda$createCores$0(BasicDistributedZkTest.java:582)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:45599

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:45599
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:516)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
	at __randomizedtesting.SeedInfo.seed([69C7C3B4D964326A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.GeneratedMethodAccessor18.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest:     1) Thread[id=3578, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=3583, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=3598, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=3588, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=3593, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=3558, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=3563, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=3603, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=3573, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=3568, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 
   1) Thread[id=3578, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=3583, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=3598, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=3588, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=3593, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=3558, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=3563, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=3603, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=3573, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=3568, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([69C7C3B4D964326A]:0)




Build Log:
[...truncated 3051 lines...]
   [junit4] Suite: org.apache.lucene.analysis.core.TestRandomChains
   [junit4]   2> Exception from random analyzer: 
   [junit4]   2> charfilters=
   [junit4]   2> tokenizer=
   [junit4]   2>   org.apache.lucene.analysis.core.LetterTokenizer()
   [junit4]   2> filters=
   [junit4]   2>   org.apache.lucene.analysis.minhash.MinHashFilter(ValidatingTokenFilter@1255a487 term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word, 3, 48, -35, true)
   [junit4]   2>   org.apache.lucene.analysis.util.ElisionFilter(ValidatingTokenFilter@7a0b7f3c term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word, [klglj])
   [junit4]   2>   org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter(ValidatingTokenFilter@8ed0d9d term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word, true)
   [junit4]   2>   org.apache.lucene.analysis.ngram.NGramTokenFilter(ValidatingTokenFilter@43c891a term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word)
   [junit4]   2> offsetsAreCorrect=true
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestRandomChains -Dtests.method=testRandomChainsWithLargeStrings -Dtests.seed=B9ED0ECBBD7CADDD -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=ru -Dtests.timezone=Africa/Monrovia -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   25.1s J0 | TestRandomChains.testRandomChainsWithLargeStrings <<<
   [junit4]    > Throwable #1: java.util.NoSuchElementException
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([B9ED0ECBBD7CADDD:D3B6B1DAE4328D2E]:0)
   [junit4]    > 	at java.util.TreeMap.key(TreeMap.java:1327)
   [junit4]    > 	at java.util.TreeMap.lastKey(TreeMap.java:297)
   [junit4]    > 	at java.util.TreeSet.last(TreeSet.java:401)
   [junit4]    > 	at org.apache.lucene.analysis.minhash.MinHashFilter$FixedSizeTreeSet.add(MinHashFilter.java:325)
   [junit4]    > 	at org.apache.lucene.analysis.minhash.MinHashFilter.incrementToken(MinHashFilter.java:159)
   [junit4]    > 	at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:67)
   [junit4]    > 	at org.apache.lucene.analysis.util.ElisionFilter.incrementToken(ElisionFilter.java:52)
   [junit4]    > 	at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:67)
   [junit4]    > 	at org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter.incrementToken(ASCIIFoldingFilter.java:102)
   [junit4]    > 	at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:67)
   [junit4]    > 	at org.apache.lucene.analysis.FilteringTokenFilter.incrementToken(FilteringTokenFilter.java:51)
   [junit4]    > 	at org.apache.lucene.analysis.ngram.NGramTokenFilter.incrementToken(NGramTokenFilter.java:98)
   [junit4]    > 	at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:67)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkResetException(BaseTokenStreamTestCase.java:405)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:510)
   [junit4]    > 	at org.apache.lucene.analysis.core.TestRandomChains.testRandomChainsWithLargeStrings(TestRandomChains.java:959)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/analysis/common/test/J0/temp/lucene.analysis.core.TestRandomChains_B9ED0ECBBD7CADDD-001
   [junit4]   2> NOTE: test params are: codec=DummyCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=DUMMY, chunkSize=8, maxDocsPerChunk=6, blockSize=176), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=DUMMY, chunkSize=8, blockSize=176)), sim=RandomSimilarity(queryNorm=false): {dummy=IB SPL-L3(800.0)}, locale=ru, timezone=Africa/Monrovia
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=93458712,total=296222720
   [junit4]   2> NOTE: All tests run in this JVM: [TestFrenchLightStemFilterFactory, TestDictionaryCompoundWordTokenFilterFactory, TestLimitTokenOffsetFilterFactory, TestHindiFilters, TestStemmer, TestPathHierarchyTokenizer, TestSpanishAnalyzer, TestFinnishLightStemFilterFactory, TestFullStrip, TestLengthFilter, TestCustomAnalyzer, TestGalicianStemFilterFactory, TestSwedishAnalyzer, TestDependencies, TestIgnore, TestTurkishAnalyzer, TestApostropheFilter, TestScandinavianFoldingFilter, TestHungarianLightStemFilterFactory, TestIndonesianStemmer, TestSpanishLightStemFilterFactory, TestElisionFilterFactory, WikipediaTokenizerTest, TestCollationKeyAnalyzer, TestGermanMinimalStemFilterFactory, TestTrimFilterFactory, TestIndonesianStemFilterFactory, TestNorwegianLightStemFilter, TestPorterStemFilter, TestRussianLightStemFilterFactory, TestCapitalizationFilter, TestArabicFilters, TestBasqueAnalyzer, TestCaseSensitive, TestCollationDocValuesField, QueryAutoStopWordAnalyzerTest, NGramTokenizerTest, TestGermanMinimalStemFilter, TestSoraniNormalizationFilterFactory, TestEnglishAnalyzer, TestPersianNormalizationFilter, TestNorwegianAnalyzer, TestCJKWidthFilter, TestDecimalDigitFilter, TestKeywordRepeatFilter, TestFactories, TestRandomChains]
   [junit4] Completed [122/272 (1!)] on J0 in 117.86s, 2 tests, 1 error <<< FAILURES!

[...truncated 8348 lines...]
   [junit4] Suite: org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/init-core-data-001
   [junit4]   2> 841040 INFO  (SUITE-ConcurrentDeleteAndCreateCollectionTest-seed#[69C7C3B4D964326A]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 841043 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testConcurrentCreateAndDeleteOverTheSameConfig
   [junit4]   2> 841044 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 841080 INFO  (Thread-953) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 841080 INFO  (Thread-953) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 841173 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.ZkTestServer start zk server on port:52340
   [junit4]   2> 841173 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 841225 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 841245 INFO  (zkCallback-503-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@1e554cc8 name:ZooKeeperConnection Watcher:127.0.0.1:52340 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 841245 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 841245 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 841245 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr/solr.xml
   [junit4]   2> 841247 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr/clusterprops.json
   [junit4]   2> 841255 INFO  (jetty-launcher-502-thread-1) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 841266 INFO  (jetty-launcher-502-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1a88dbc2{/solr,null,AVAILABLE}
   [junit4]   2> 841269 INFO  (jetty-launcher-502-thread-1) [    ] o.e.j.s.ServerConnector Started ServerConnector@1923033d{SSL,[ssl, http/1.1]}{127.0.0.1:44324}
   [junit4]   2> 841269 INFO  (jetty-launcher-502-thread-1) [    ] o.e.j.s.Server Started @849329ms
   [junit4]   2> 841269 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=44324}
   [junit4]   2> 841293 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 841293 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1'
   [junit4]   2> 841294 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 841294 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 841294 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 841309 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 841353 INFO  (zkCallback-504-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@c016771 name:ZooKeeperConnection Watcher:127.0.0.1:52340/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 841355 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 841355 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 841355 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 841369 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1/.
   [junit4]   2> 841369 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.CoreContainer New CoreContainer 866240293
   [junit4]   2> 841369 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1]
   [junit4]   2> 841369 WARN  (jetty-launcher-502-thread-1) [    ] o.a.s.c.CoreContainer Couldn't add files from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1/lib to classpath: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1/lib
   [junit4]   2> 841390 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : https,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,
   [junit4]   2> 841414 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.u.UpdateShardHandler Created UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&maxConnections=100000&maxConnectionsPerHost=100000
   [junit4]   2> 841414 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 841414 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 841414 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52340/solr
   [junit4]   2> 841414 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 841414 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 841437 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 841452 INFO  (zkCallback-507-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6695662e name:ZooKeeperConnection Watcher:127.0.0.1:52340 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 841452 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 841452 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 841453 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$83/793159416@191f8e5d
   [junit4]   2> 841501 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 841517 INFO  (zkCallback-508-thread-1-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@513e6957 name:ZooKeeperConnection Watcher:127.0.0.1:52340/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 841517 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 841518 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
   [junit4]   2> 841520 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
   [junit4]   2> 841523 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
   [junit4]   2> 841524 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
   [junit4]   2> 841526 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
   [junit4]   2> 841527 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 841528 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 841529 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 841530 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 841531 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 841532 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 841532 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ZkStateReader Loaded cluster properties: {urlScheme=https}
   [junit4]   2> 841533 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 841535 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
   [junit4]   2> 841536 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
   [junit4]   2> 841537 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 841538 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/96575435014995971-127.0.0.1:44324_solr-n_0000000000
   [junit4]   2> 841538 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:44324_solr
   [junit4]   2> 841538 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
   [junit4]   2> 841539 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.Overseer Overseer (id=96575435014995971-127.0.0.1:44324_solr-n_0000000000) starting
   [junit4]   2> 841541 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
   [junit4]   2> 841546 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 841561 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.Overseer Starting to work on the main queue
   [junit4]   2> 841562 INFO  (OverseerCollectionConfigSetProcessor-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
   [junit4]   2> 842034 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 842035 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44324_solr
   [junit4]   2> 842035 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:44324_solr
   [junit4]   2> 842037 INFO  (zkCallback-508-thread-1-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 842037 INFO  (zkCallback-508-thread-1-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 842200 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 842200 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 842225 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1/.
   [junit4]   2> 842226 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions
   [junit4]   2> 842233 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.s.SolrDispatchFilter user.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1
   [junit4]   2> 842233 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 842233 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 842245 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 842257 INFO  (zkCallback-510-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@64b90cc9 name:ZooKeeperConnection Watcher:127.0.0.1:52340 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 842257 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 842257 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 842262 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 842300 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 842310 INFO  (zkCallback-512-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@1c09f4f2 name:ZooKeeperConnection Watcher:127.0.0.1:52340/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 842310 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 842311 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 842311 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/testconfig/schema.xml
   [junit4]   2> 842314 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/testconfig/solrconfig.xml
   [junit4]   2> 842505 INFO  (qtp946450217-3432) [n:127.0.0.1:44324_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=testconfig&name=collection0&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 842508 INFO  (OverseerCollectionConfigSetProcessor-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2>   "name":"collection0",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"testconfig",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 842509 INFO  (OverseerThreadFactory-1293-thread-1-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"collection0",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"testconfig",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 842510 INFO  (OverseerThreadFactory-1293-thread-1-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection0
   [junit4]   2> 842510 INFO  (qtp946450217-3433) [n:127.0.0.1:44324_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=testconfig&name=collection1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 842517 INFO  (OverseerThreadFactory-1293-thread-1-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection0 
   [junit4]   2> 842533 INFO  (OverseerThreadFactory-1293-thread-1-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection0
   [junit4]   2> 842538 INFO  (OverseerCollectionConfigSetProcessor-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000002 message:{
   [junit4]   2>   "name":"collection1",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"testconfig",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 842540 INFO  (OverseerThreadFactory-1293-thread-2-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"collection1",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"testconfig",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 842540 INFO  (OverseerThreadFactory-1293-thread-2-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 842541 INFO  (OverseerThreadFactory-1293-thread-2-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection1 
   [junit4]   2> 842545 INFO  (zkCallback-508-thread-1-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 842545 INFO  (OverseerThreadFactory-1293-thread-2-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1
   [junit4]   2> 842562 INFO  (zkCallback-508-thread-2-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 842580 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
   [junit4]   2>   "name":"collection1",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"testconfig",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 842580 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection1
   [junit4]   2> 842586 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection1/state.json
   [junit4]   2> 842588 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "name":"collection0",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"testconfig",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 842589 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection0
   [junit4]   2> 842777 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection0/state.json
   [junit4]   2> 842778 INFO  (OverseerThreadFactory-1293-thread-2-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.CreateCollectionCmd Creating SolrCores for new collection collection1, shardNames [shard1] , replicationFactor : 1
   [junit4]   2> 842778 INFO  (OverseerThreadFactory-1293-thread-2-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.CreateCollectionCmd Creating core collection1_shard1_replica1 as part of shard shard1 of collection collection1 on 127.0.0.1:44324_solr
   [junit4]   2> 842885 INFO  (OverseerThreadFactory-1293-thread-1-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.CreateCollectionCmd Creating SolrCores for new collection collection0, shardNames [shard1] , replicationFactor : 1
   [junit4]   2> 842885 INFO  (OverseerThreadFactory-1293-thread-1-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.CreateCollectionCmd Creating core collection0_shard1_replica1 as part of shard shard1 of collection collection0 on 127.0.0.1:44324_solr
   [junit4]   2> 842962 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=testconfig&name=collection0_shard1_replica1&action=CREATE&numShards=1&collection=collection0&shard=shard1&wt=javabin&version=2
   [junit4]   2> 842964 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection0_shard1_replica1, collection.configName=testconfig, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.json, collection=collection0, numShards=1, dataDir=data/, shard=shard1}
   [junit4]   2> 842964 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.ZkController publishing state=down
   [junit4]   2> 842965 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.ZkController Registering watch for collection collection0
   [junit4]   2> 842966 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.c.ZkStateReader Add data for [collection0] ver [0]
   [junit4]   2> 842966 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.ZkController look for our core node name
   [junit4]   2> 842978 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=testconfig&name=collection1_shard1_replica1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2
   [junit4]   2> 842978 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection1_shard1_replica1, collection.configName=testconfig, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.json, collection=collection1, numShards=1, dataDir=data/, shard=shard1}
   [junit4]   2> 842979 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.ZkController publishing state=down
   [junit4]   2> 842980 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.ZkController Registering watch for collection collection1
   [junit4]   2> 842980 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.c.ZkStateReader Add data for [collection1] ver [0]
   [junit4]   2> 842980 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.ZkController look for our core node name
   [junit4]   2> 842997 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
   [junit4]   2>   "core":"collection0_shard1_replica1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"https://127.0.0.1:44324/solr",
   [junit4]   2>   "node_name":"127.0.0.1:44324_solr",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"collection0",
   [junit4]   2>   "operation":"state"} current state version: 0
   [junit4]   2> 842997 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
   [junit4]   2>   "core":"collection0_shard1_replica1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"https://127.0.0.1:44324/solr",
   [junit4]   2>   "node_name":"127.0.0.1:44324_solr",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"collection0",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 843002 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
   [junit4]   2>   "core":"collection1_shard1_replica1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"https://127.0.0.1:44324/solr",
   [junit4]   2>   "node_name":"127.0.0.1:44324_solr",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "operation":"state"} current state version: 0
   [junit4]   2> 843003 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
   [junit4]   2>   "core":"collection1_shard1_replica1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"https://127.0.0.1:44324/solr",
   [junit4]   2>   "node_name":"127.0.0.1:44324_solr",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 843003 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/collection0/state.json version: 0
   [junit4]   2> 843005 INFO  (zkCallback-508-thread-4-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 843009 INFO  (zkCallback-508-thread-4-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ZkStateReader Updating data for [collection0] from [0] to [1]
   [junit4]   2> 843178 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 843179 INFO  (zkCallback-508-thread-3-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 843179 INFO  (zkCallback-508-thread-3-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ZkStateReader Updating data for [collection1] from [0] to [1]
   [junit4]   2> 844030 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.ZkController waiting to find shard id in clusterstate for collection0_shard1_replica1
   [junit4]   2> 844030 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.ZkController Check for collection zkNode:collection0
   [junit4]   2> 844030 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.ZkController waiting to find shard id in clusterstate for collection1_shard1_replica1
   [junit4]   2> 844030 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.ZkController Check for collection zkNode:collection1
   [junit4]   2> 844031 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.ZkController Collection zkNode exists
   [junit4]   2> 844032 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.c.ZkStateReader Load collection config from: [/collections/collection1]
   [junit4]   2> 844032 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.ZkController Collection zkNode exists
   [junit4]   2> 844032 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.c.ZkStateReader Load collection config from: [/collections/collection0]
   [junit4]   2> 844033 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.c.ZkStateReader path=[/collections/collection1] [configName]=[testconfig] specified config exists in ZooKeeper
   [junit4]   2> 844033 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1/collection1_shard1_replica1'
   [junit4]   2> 844034 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 844034 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 844034 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.c.ZkStateReader path=[/collections/collection0] [configName]=[testconfig] specified config exists in ZooKeeper
   [junit4]   2> 844035 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1/collection0_shard1_replica1'
   [junit4]   2> 844035 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 844035 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 844036 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.Config loaded config solrconfig.xml with version 0 
   [junit4]   2> 844037 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.Config loaded config solrconfig.xml with version 0 
   [junit4]   2> 844042 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 844043 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 844050 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 844061 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 844094 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 844107 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 844119 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.s.IndexSchema [collection0_shard1_replica1] Schema name=minimal
   [junit4]   2> 844122 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 844124 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.IndexSchema [collection1_shard1_replica1] Schema name=minimal
   [junit4]   2> 844126 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 844126 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properties: Can't find resource 'configsetprops.json' in classpath or '/configs/testconfig', cwd=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1
   [junit4]   2> 844126 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'collection0_shard1_replica1' using configuration from collection collection0
   [junit4]   2> 844127 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 844129 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore [[collection0_shard1_replica1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1/collection0_shard1_replica1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1/./collection0_shard1_replica1/data/]
   [junit4]   2> 844129 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore JMX monitoring not detected for core: collection0_shard1_replica1
   [junit4]   2> 844129 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1/./collection0_shard1_replica1/data/snapshot_metadata
   [junit4]   2> 844129 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.s.SolrSnapshotMetaDataManager Loading from snapshot metadata file...
   [junit4]   2> 844130 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1/./collection0_shard1_replica1/data
   [junit4]   2> 844130 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore New index directory detected: old=null new=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1/./collection0_shard1_replica1/data/index/
   [junit4]   2> 844130 WARN  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore [collection0_shard1_replica1] Solr index directory '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1/./collection0_shard1_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 844130 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-sla

[...truncated too long message...]

p/solr.handler.TestSolrConfigHandlerCloud_69C7C3B4D964326A-001/shard-4-001/cores/collection1/data/snapshot_metadata
   [junit4]   2> 7363461 INFO  (Thread-87801) [n:127.0.0.1:49246_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.CachingDirectoryFactory looking to close /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestSolrConfigHandlerCloud_69C7C3B4D964326A-001/shard-4-001/cores/collection1/data [CachedDir<<refCount=0;path=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestSolrConfigHandlerCloud_69C7C3B4D964326A-001/shard-4-001/cores/collection1/data;done=false>>]
   [junit4]   2> 7363461 INFO  (Thread-87801) [n:127.0.0.1:49246_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.CachingDirectoryFactory Closing directory, CoreContainer#isShutdown=true
   [junit4]   2> 7363461 INFO  (Thread-87801) [n:127.0.0.1:49246_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.CachingDirectoryFactory Closing directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestSolrConfigHandlerCloud_69C7C3B4D964326A-001/shard-4-001/cores/collection1/data
   [junit4]   2> 7363467 INFO  (Thread-87801) [n:127.0.0.1:49246_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.CachingDirectoryFactory looking to close /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestSolrConfigHandlerCloud_69C7C3B4D964326A-001/shard-4-001/cores/collection1/data/index [CachedDir<<refCount=0;path=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestSolrConfigHandlerCloud_69C7C3B4D964326A-001/shard-4-001/cores/collection1/data/index;done=false>>]
   [junit4]   2> 7363467 INFO  (Thread-87801) [n:127.0.0.1:49246_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.CachingDirectoryFactory Closing directory, CoreContainer#isShutdown=true
   [junit4]   2> 7363467 INFO  (Thread-87801) [n:127.0.0.1:49246_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.CachingDirectoryFactory Closing directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestSolrConfigHandlerCloud_69C7C3B4D964326A-001/shard-4-001/cores/collection1/data/index
   [junit4]   2> 7363467 WARN  (Thread-87801) [n:127.0.0.1:49246_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.SolrCores Tried to remove core collection1 from pendingCoreOps and it wasn't there. 
   [junit4]   2> 7363467 WARN  (Thread-87801) [n:127.0.0.1:49246_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.ZkController listener throws error
   [junit4]   2> org.apache.solr.common.SolrException: Unable to reload core [collection1]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:968)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.lambda$getConfListener$6(SolrCore.java:2602)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController.lambda$fireEventListeners$3(ZkController.java:2443)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Failed to load config name for collection:collection1 due to: 
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:833)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.reload(SolrCore.java:503)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:962)
   [junit4]   2> 	... 3 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Failed to load config name for collection:collection1 due to: 
   [junit4]   2> 	at org.apache.solr.rest.ManagedResourceStorage.newStorageIO(ManagedResourceStorage.java:98)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.initRestManager(SolrCore.java:2382)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:809)
   [junit4]   2> 	... 5 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Error loading config name for collection collection1
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.readConfigName(ZkStateReader.java:197)
   [junit4]   2> 	at org.apache.solr.rest.ManagedResourceStorage.newStorageIO(ManagedResourceStorage.java:95)
   [junit4]   2> 	... 7 more
   [junit4]   2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:348)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:345)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.readConfigName(ZkStateReader.java:178)
   [junit4]   2> 	... 8 more
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {}, docValues:{}, maxPointsInLeafNode=238, maxMBSortInHeap=5.750380438790871, sim=RandomSimilarity(queryNorm=false): {}, locale=ko, timezone=Africa/Niamey
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=157937416,total=521142272
   [junit4]   2> NOTE: All tests run in this JVM: [TestSolrConfigHandler, TestLegacyNumericRangeQueryBuilder, AnalyticsQueryTest, ResourceLoaderTest, TestRandomCollapseQParserPlugin, TestDFISimilarityFactory, PeerSyncTest, SmileWriterTest, TestSortByMinMaxFunction, RankQueryTest, DateFieldTest, LeaderElectionIntegrationTest, TestFieldTypeCollectionResource, DistributedExpandComponentTest, QueryEqualityTest, TestJsonFacets, QueryElevationComponentTest, RestartWhileUpdatingTest, SpatialRPTFieldTypeTest, TestSolrDeletionPolicy2, SolrRequestParserTest, HdfsTlogReplayBufferedWhileIndexingTest, UUIDUpdateProcessorFallbackTest, OverseerTaskQueueTest, CacheHeaderTest, TestDocTermOrds, CachingDirectoryFactoryTest, TestSimpleQParserPlugin, TestSerializedLuceneMatchVersion, SolrCloudExampleTest, TestConfigSetProperties, TestReplicationHandler, CloudMLTQParserTest, SignatureUpdateProcessorFactoryTest, XsltUpdateRequestHandlerTest, TestOmitPositions, ConnectionManagerTest, HdfsNNFailoverTest, DistributedFacetPivotLongTailTest, TestStressRecovery, TestRangeQuery, SecurityConfHandlerTest, RequiredFieldsTest, DataDrivenBlockJoinTest, TestHdfsUpdateLog, DistribCursorPagingTest, TestImplicitCoreProperties, TestComponentsName, HttpPartitionTest, TestUniqueKeyFieldResource, BlockJoinFacetRandomTest, TestCrossCoreJoin, TestConfigSetsAPIExclusivity, DistributedSuggestComponentTest, OverseerModifyCollectionTest, TestPHPSerializedResponseWriter, TestAtomicUpdateErrorCases, TestHighlightDedupGrouping, StandardRequestHandlerTest, DateRangeFieldTest, CollectionsAPISolrJTest, TestReplicaProperties, TestSolr4Spatial, SyncSliceTest, AsyncCallRequestStatusResponseTest, ZkStateWriterTest, TestExactSharedStatsCache, TestScoreJoinQPNoScore, TestJoin, TestManagedSchema, TestDocumentBuilder, RegexBoostProcessorTest, TestHighFrequencyDictionaryFactory, SuggestComponentContextFilterQueryTest, TestUseDocValuesAsStored, JavabinLoaderTest, DeleteReplicaTest, TestJmxIntegration, TestExpandComponent, TestHdfsCloudBackupRestore, StressHdfsTest, TestSSLRandomization, AlternateDirectoryTest, DistributedQueueTest, TestDynamicFieldResource, TestSolrCoreProperties, HLLUtilTest, TestConfigReload, TestDistributedSearch, TestGraphTermsQParserPlugin, TestReqParamsAPI, ResponseHeaderTest, TestValueSourceCache, TestQuerySenderNoQuery, SaslZkACLProviderTest, TestFoldingMultitermQuery, PathHierarchyTokenizerFactoryTest, CursorPagingTest, TestSystemIdResolver, SpellingQueryConverterTest, ClusterStateTest, ZkNodePropsTest, SliceStateTest, SystemInfoHandlerTest, UUIDFieldTest, DistributedMLTComponentTest, TestRTGBase, DistributedIntervalFacetingTest, TestEmbeddedSolrServerConstructors, TestJettySolrRunner, ActionThrottleTest, AsyncMigrateRouteKeyTest, CdcrReplicationDistributedZkTest, TestStressLiveNodes, TestBackupRepositoryFactory, TestConfigOverlay, TestConfigSets, TestNRTOpen, RequestLoggingTest, TestBlobHandler, TestSolrConfigHandlerCloud]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSolrConfigHandlerCloud -Dtests.seed=69C7C3B4D964326A -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=ko -Dtests.timezone=Africa/Niamey -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J0 | TestSolrConfigHandlerCloud (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: ObjectTracker found 8 object(s) that were not released!!! [MDCAwareThreadPoolExecutor, MockDirectoryWrapper, SolrCore, MockDirectoryWrapper, MDCAwareThreadPoolExecutor, SolrCore, MDCAwareThreadPoolExecutor, MockDirectoryWrapper]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([69C7C3B4D964326A]:0)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4] Completed [544/631 (6!)] on J0 in 114.89s, 1 test, 1 failure <<< FAILURES!

[...truncated 60452 lines...]



[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1113 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1113/

13 tests failed.
FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard2

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard2
	at __randomizedtesting.SeedInfo.seed([8F24B045D9638E9:54EFAC8593B68CDF]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart(CdcrReplicationDistributedZkTest.java:236)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.DistributedVersionInfoTest.testReplicaVersionHandling

Error Message:
Captured an uncaught exception in thread: Thread[id=169355, name=Thread-5485, state=RUNNABLE, group=TGRP-DistributedVersionInfoTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=169355, name=Thread-5485, state=RUNNABLE, group=TGRP-DistributedVersionInfoTest]
	at __randomizedtesting.SeedInfo.seed([8F24B045D9638E9:D40B9CFEFFEDF2A8]:0)
Caused by: java.lang.IllegalArgumentException: bound must be positive
	at __randomizedtesting.SeedInfo.seed([8F24B045D9638E9]:0)
	at java.util.Random.nextInt(Random.java:388)
	at org.apache.solr.cloud.DistributedVersionInfoTest$3.run(DistributedVersionInfoTest.java:204)


FAILED:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:36753/na/collection2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:36753/na/collection2
	at __randomizedtesting.SeedInfo.seed([8F24B045D9638E9:80A674DEF36A5511]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:415)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:367)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1280)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1050)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:992)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:484)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:463)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.commit(AbstractFullDistribZkTestBase.java:1530)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:664)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test(FullSolrCloudDistribCmdsTest.java:152)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 54 more


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:38312/solrj_collection4, http://127.0.0.1:58988/solrj_collection4, http://127.0.0.1:43269/solrj_collection4]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:38312/solrj_collection4, http://127.0.0.1:58988/solrj_collection4, http://127.0.0.1:43269/solrj_collection4]
	at __randomizedtesting.SeedInfo.seed([8F24B045D9638E9:80A674DEF36A5511]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:394)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1280)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1050)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:992)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.queryAndAssertResultSize(SharedFSAutoReplicaFailoverTest.java:326)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:260)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:127)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:43269/solrj_collection4: no servers hosting shard: shard2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:608)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:415)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:367)
	... 50 more


FAILED:  junit.framework.TestSuite.org.apache.solr.handler.TestSolrConfigHandlerCloud

Error Message:
ObjectTracker found 4 object(s) that were not released!!! [MockDirectoryWrapper, MockDirectoryWrapper, MockDirectoryWrapper, MDCAwareThreadPoolExecutor]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 4 object(s) that were not released!!! [MockDirectoryWrapper, MockDirectoryWrapper, MockDirectoryWrapper, MDCAwareThreadPoolExecutor]
	at __randomizedtesting.SeedInfo.seed([8F24B045D9638E9]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.GeneratedMethodAccessor66.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest.test

Error Message:
Error from server at http://127.0.0.1:33658/p/c: Error CREATEing SolrCore 'test_unload_shard_and_collection_2': Unable to create core [test_unload_shard_and_collection_2] Caused by: Direct buffer memory

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:33658/p/c: Error CREATEing SolrCore 'test_unload_shard_and_collection_2': Unable to create core [test_unload_shard_and_collection_2] Caused by: Direct buffer memory
	at __randomizedtesting.SeedInfo.seed([8F24B045D9638E9:80A674DEF36A5511]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:608)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.UnloadDistributedZkTest.testUnloadShardAndCollection(UnloadDistributedZkTest.java:137)
	at org.apache.solr.cloud.UnloadDistributedZkTest.test(UnloadDistributedZkTest.java:70)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigAliasReplication

Error Message:
expected:<1> but was:<0>

Stack Trace:
java.lang.AssertionError: expected:<1> but was:<0>
	at __randomizedtesting.SeedInfo.seed([8F24B045D9638E9:FF81A55C9B7E970F]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigAliasReplication(TestReplicationHandler.java:1329)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestStressReplication

Error Message:
timed out waiting for collection1 startAt time to exceed: Sat Sep 10 12:13:15 EDT 2016

Stack Trace:
java.lang.AssertionError: timed out waiting for collection1 startAt time to exceed: Sat Sep 10 12:13:15 EDT 2016
	at __randomizedtesting.SeedInfo.seed([8F24B045D9638E9:D3594BC258BE515A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.handler.TestReplicationHandler.watchCoreStartAt(TestReplicationHandler.java:1508)
	at org.apache.solr.handler.TestReplicationHandler.doTestStressReplication(TestReplicationHandler.java:858)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchAddsWithDelete

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard2

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard2
	at __randomizedtesting.SeedInfo.seed([8F24B045D9638E9:7FE832A0F58F39C5]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchAddsWithDelete(CdcrReplicationDistributedZkTest.java:526)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:50004

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:50004
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:497)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:44483

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:44483
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:516)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
	at __randomizedtesting.SeedInfo.seed([8F24B045D9638E9]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.GeneratedMethodAccessor66.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest:     1) Thread[id=70872, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=70862, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=70907, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=70887, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=70897, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=70867, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=70882, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=70877, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=70902, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=70892, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 
   1) Thread[id=70872, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=70862, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=70907, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=70887, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=70897, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=70867, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=70882, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=70877, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=70902, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=70892, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([8F24B045D9638E9]:0)




Build Log:
[...truncated 11384 lines...]
   [junit4] Suite: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_8F24B045D9638E9-001/init-core-data-001
   [junit4]   2> 547883 INFO  (SUITE-FullSolrCloudDistribCmdsTest-seed#[8F24B045D9638E9]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 547884 INFO  (SUITE-FullSolrCloudDistribCmdsTest-seed#[8F24B045D9638E9]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /na/
   [junit4]   2> 547885 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 547935 INFO  (Thread-1450) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 547935 INFO  (Thread-1450) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 548039 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.ZkTestServer start zk server on port:48470
   [junit4]   2> 548039 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 548063 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 548143 INFO  (zkCallback-502-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@34a87f8d name:ZooKeeperConnection Watcher:127.0.0.1:48470 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 548147 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 548147 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 548147 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 548150 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 548151 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 548153 INFO  (zkCallback-503-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@7dc1d05e name:ZooKeeperConnection Watcher:127.0.0.1:48470/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 548153 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 548153 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 548153 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1
   [junit4]   2> 548154 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/shards
   [junit4]   2> 548218 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection
   [junit4]   2> 548219 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection/shards
   [junit4]   2> 548220 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 548220 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 548222 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 548222 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/schema.xml
   [junit4]   2> 548223 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 548223 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 548224 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 548225 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 548226 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 548226 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/protwords.txt
   [junit4]   2> 548227 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 548227 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/currency.xml
   [junit4]   2> 548228 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 548228 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 548229 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 548229 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 548230 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 548231 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 548232 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 548232 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 548233 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 548233 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 548440 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_8F24B045D9638E9-001/control-001/cores/collection1
   [junit4]   2> 548442 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 548477 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@59d040ea{/na,null,AVAILABLE}
   [junit4]   2> 548494 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.e.j.s.ServerConnector Started ServerConnector@30567dde{HTTP/1.1,[http/1.1]}{127.0.0.1:33093}
   [junit4]   2> 548495 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.e.j.s.Server Started @556035ms
   [junit4]   2> 548495 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_8F24B045D9638E9-001/tempDir-001/control/data, hostContext=/na, hostPort=33093, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_8F24B045D9638E9-001/control-001/cores}
   [junit4]   2> 548499 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 548499 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_8F24B045D9638E9-001/control-001'
   [junit4]   2> 548499 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 548499 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 548500 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 548531 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 548531 INFO  (zkCallback-504-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3704b9ac name:ZooKeeperConnection Watcher:127.0.0.1:48470/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 548531 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 548532 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 548543 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 548543 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_8F24B045D9638E9-001/control-001/solr.xml
   [junit4]   2> 548548 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_8F24B045D9638E9-001/control-001/cores
   [junit4]   2> 548548 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.CoreContainer New CoreContainer 703387634
   [junit4]   2> 548548 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_8F24B045D9638E9-001/control-001]
   [junit4]   2> 548548 WARN  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.CoreContainer Couldn't add files from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_8F24B045D9638E9-001/control-001/lib to classpath: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_8F24B045D9638E9-001/control-001/lib
   [junit4]   2> 548549 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,
   [junit4]   2> 548603 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.u.UpdateShardHandler Created UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&maxConnections=100000&maxConnectionsPerHost=100000
   [junit4]   2> 548603 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 548603 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 548603 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:48470/solr
   [junit4]   2> 548604 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 548604 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 548619 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 548631 INFO  (zkCallback-507-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4f62f01f name:ZooKeeperConnection Watcher:127.0.0.1:48470 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 548636 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 548636 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 548637 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$99/841033882@11d68def
   [junit4]   2> 548804 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 548816 INFO  (zkCallback-508-thread-1-processing-n:127.0.0.1:33093_na) [n:127.0.0.1:33093_na    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6d43343f name:ZooKeeperConnection Watcher:127.0.0.1:48470/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 548816 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 548817 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
   [junit4]   2> 548819 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
   [junit4]   2> 548821 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
   [junit4]   2> 548822 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
   [junit4]   2> 548823 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
   [junit4]   2> 548825 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 548826 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 548826 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 548827 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 548828 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 548828 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 548829 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 548831 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
   [junit4]   2> 548832 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
   [junit4]   2> 548833 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 548833 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/96568706267938820-127.0.0.1:33093_na-n_0000000000
   [junit4]   2> 548833 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:33093_na
   [junit4]   2> 548833 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
   [junit4]   2> 548834 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.Overseer Overseer (id=96568706267938820-127.0.0.1:33093_na-n_0000000000) starting
   [junit4]   2> 548835 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
   [junit4]   2> 548839 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=10000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 548847 INFO  (OverseerStateUpdate-96568706267938820-127.0.0.1:33093_na-n_0000000000) [n:127.0.0.1:33093_na    ] o.a.s.c.Overseer Starting to work on the main queue
   [junit4]   2> 548848 INFO  (OverseerCollectionConfigSetProcessor-96568706267938820-127.0.0.1:33093_na-n_0000000000) [n:127.0.0.1:33093_na    ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
   [junit4]   2> 548864 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33093_na
   [junit4]   2> 548864 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:33093_na
   [junit4]   2> 548865 INFO  (OverseerStateUpdate-96568706267938820-127.0.0.1:33093_na-n_0000000000) [n:127.0.0.1:33093_na    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 548866 INFO  (zkCallback-508-thread-1-processing-n:127.0.0.1:33093_na) [n:127.0.0.1:33093_na    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 548867 INFO  (zkCallback-508-thread-1-processing-n:127.0.0.1:33093_na) [n:127.0.0.1:33093_na    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 548904 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 548904 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 548905 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_8F24B045D9638E9-001/control-001/cores
   [junit4]   2> 548906 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.json, collection=control_collection, coreNodeName=, dataDir=data/, shard=}
  

[...truncated too long message...]

un.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:430)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
   [junit4]   2> 	at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
   [junit4]   2> 	at org.apache.hadoop.metrics2.util.MBeans.unregister(MBeans.java:81)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stopMBeans(MetricsSourceAdapter.java:226)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stop(MetricsSourceAdapter.java:211)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stopSources(MetricsSystemImpl.java:463)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
   [junit4]   2> 	at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
   [junit4]   2> 	at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
   [junit4]   2> 	at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:198)
   [junit4]   2> 	at org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.teardownClass(HdfsCollectionsAPIDistributedZkTest.java:52)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: java.lang.NullPointerException
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
   [junit4]   2> 	... 54 more
   [junit4]   2> 13249318 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[8F24B045D9638E9]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_8F24B045D9638E9-001
   [junit4]   2> Sep 10, 2016 7:03:41 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 35 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=8, maxDocsPerChunk=237, blockSize=396), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=8, blockSize=396)), sim=RandomSimilarity(queryNorm=false): {}, locale=es-CL, timezone=Antarctica/Casey
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=2,free=141441624,total=516423680
   [junit4]   2> NOTE: All tests run in this JVM: [TestBulkSchemaConcurrent, TestFieldCollectionResource, TestFieldCache, CdcrReplicationDistributedZkTest, DirectSolrConnectionTest, SolrIndexConfigTest, ClassificationUpdateProcessorFactoryTest, TestLRUCache, DeleteShardTest, HdfsBasicDistributedZk2Test, TestDefaultSearchFieldResource, TestRandomRequestDistribution, SimpleCollectionCreateDeleteTest, DocumentBuilderTest, TestNamedUpdateProcessors, TestHashPartitioner, TestSolrConfigHandlerConcurrent, TestBM25SimilarityFactory, TestDistributedGrouping, TestFastOutputStream, SolrIndexSplitterTest, TestSchemaSimilarityResource, TestCSVResponseWriter, TestFieldCacheWithThreads, TestMiniSolrCloudClusterSSL, ShardRoutingCustomTest, FileUtilsTest, TestSystemIdResolver, TermsComponentTest, TestMinMaxOnMultiValuedField, TestSolrCloudWithKerberosAlt, TlogReplayBufferedWhileIndexingTest, AliasIntegrationTest, BlockCacheTest, TestSolrQueryResponse, DirectSolrSpellCheckerTest, TestScoreJoinQPScore, MultiTermTest, TestFieldSortValues, TestGraphMLResponseWriter, MigrateRouteKeyTest, SimplePostToolTest, CreateCollectionCleanupTest, CSVRequestHandlerTest, TestSurroundQueryParser, TestQueryTypes, TestUpdate, SampleTest, CursorMarkTest, TestRuleBasedAuthorizationPlugin, URLClassifyProcessorTest, TestManagedStopFilterFactory, TestRequestForwarding, TestFiltering, BooleanFieldTest, SpellCheckCollatorTest, LeaderFailoverAfterPartitionTest, BigEndianAscendingWordSerializerTest, TestFieldTypeResource, SolrXmlInZkTest, FullHLLTest, DistanceUnitsTest, CollectionsAPIDistributedZkTest, DeleteLastCustomShardedReplicaTest, BlockJoinFacetSimpleTest, SliceStateTest, TestSweetSpotSimilarityFactory, SpellingQueryConverterTest, CollectionTooManyReplicasTest, TestOverriddenPrefixQueryForCustomFieldType, CoreSorterTest, PKIAuthenticationIntegrationTest, TestReloadAndDeleteDocs, DistributedFacetPivotSmallAdvancedTest, FacetPivotSmallTest, TestInfoStreamLogging, DistributedVersionInfoTest, DistribDocExpirationUpdateProcessorTest, ReplaceNodeTest, TestLMDirichletSimilarityFactory, SpatialFilterTest, RequestLoggingTest, SolrCmdDistributorTest, SparseHLLTest, SearchHandlerTest, AsyncMigrateRouteKeyTest, TestManagedSchemaAPI, MoreLikeThisHandlerTest, OverriddenZkACLAndCredentialsProvidersTest, HdfsSyncSliceTest, CoreMergeIndexesAdminHandlerTest, DistanceFunctionTest, IndexSchemaRuntimeFieldTest, AnalysisErrorHandlingTest, SortByFunctionTest, RAMDirectoryFactoryTest, TestCloudInspectUtil, HdfsChaosMonkeySafeLeaderTest, SoftAutoCommitTest, SegmentsInfoRequestHandlerTest, SchemaVersionSpecificBehaviorTest, TestRTGBase, ConvertedLegacyTest, TestSolrCLIRunExample, HdfsCollectionsAPIDistributedZkTest]
   [junit4] Completed [631/631 (10!)] on J1 in 7031.62s, 1 test, 1 error <<< FAILURES!

[...truncated 1 lines...]
   [junit4] JVM J1: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20160910_152243_423.sysout
   [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: Java heap space
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid6780.hprof ...
   [junit4] Heap dump file created [722169601 bytes in 63.845 secs]
   [junit4] <<< JVM J1: EOF ----

[...truncated 11040 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:715: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid6762.hprof
* java_pid6780.hprof

Total time: 506 minutes 35 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1112 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1112/

12 tests failed.
FAILED:  org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest.test

Error Message:
Error from server at http://127.0.0.1:46438: Error CREATEing SolrCore 'test_unload_shard_and_collection_1': Unable to create core [test_unload_shard_and_collection_1] Caused by: Direct buffer memory

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:46438: Error CREATEing SolrCore 'test_unload_shard_and_collection_1': Unable to create core [test_unload_shard_and_collection_1] Caused by: Direct buffer memory
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:608)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.UnloadDistributedZkTest.testUnloadShardAndCollection(UnloadDistributedZkTest.java:129)
	at org.apache.solr.cloud.UnloadDistributedZkTest.test(UnloadDistributedZkTest.java:70)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.core.OpenCloseCoreStressTest.test10Minutes

Error Message:
Captured an uncaught exception in thread: Thread[id=17412, name=Thread-7291, state=RUNNABLE, group=TGRP-OpenCloseCoreStressTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=17412, name=Thread-7291, state=RUNNABLE, group=TGRP-OpenCloseCoreStressTest]
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigAliasReplication

Error Message:
expected:<1> but was:<0>

Stack Trace:
java.lang.AssertionError: expected:<1> but was:<0>
	at __randomizedtesting.SeedInfo.seed([327C025A1D389D1D:C50FEC02DBD032FB]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigAliasReplication(TestReplicationHandler.java:1329)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestStressReplication

Error Message:
[index.20160909000326436, replication.properties, index.properties, index.20160909000333505, snapshot_metadata] expected:<1> but was:<2>

Stack Trace:
java.lang.AssertionError: [index.20160909000326436, replication.properties, index.properties, index.20160909000333505, snapshot_metadata] expected:<1> but was:<2>
	at __randomizedtesting.SeedInfo.seed([327C025A1D389D1D:E9D7029C1810F4AE]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.apache.solr.handler.TestReplicationHandler.checkForSingleIndex(TestReplicationHandler.java:907)
	at org.apache.solr.handler.TestReplicationHandler.doTestStressReplication(TestReplicationHandler.java:874)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testResilienceWithDeleteByQueryOnTarget

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard2

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard2
	at __randomizedtesting.SeedInfo.seed([327C025A1D389D1D:921E90C0CD615858]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testResilienceWithDeleteByQueryOnTarget(CdcrReplicationDistributedZkTest.java:589)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterLeaderChange

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard2

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard2
	at __randomizedtesting.SeedInfo.seed([327C025A1D389D1D:E08C4EB943973B2F]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterLeaderChange(CdcrReplicationDistributedZkTest.java:306)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationStartStop

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard1

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard1
	at __randomizedtesting.SeedInfo.seed([327C025A1D389D1D:B1BFDCC909E01694]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationStartStop(CdcrReplicationDistributedZkTest.java:196)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchAddsWithDelete

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard2

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard2
	at __randomizedtesting.SeedInfo.seed([327C025A1D389D1D:45667BFEB5219C31]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchAddsWithDelete(CdcrReplicationDistributedZkTest.java:526)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:59906/vo/yy

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:59906/vo/yy
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:516)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:47352

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:47352
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:516)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
	at __randomizedtesting.SeedInfo.seed([327C025A1D389D1D]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest:     1) Thread[id=73355, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=73350, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=73340, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=73330, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=73335, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=73320, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=73325, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=73315, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=73345, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=73360, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 
   1) Thread[id=73355, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=73350, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=73340, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=73330, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=73335, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=73320, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=73325, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=73315, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=73345, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=73360, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([327C025A1D389D1D]:0)




Build Log:
[...truncated 11251 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_327C025A1D389D1D-001/init-core-data-001
   [junit4]   2> 93531 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[327C025A1D389D1D]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 93531 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[327C025A1D389D1D]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 93567 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 93574 INFO  (Thread-82) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 93579 INFO  (Thread-82) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 94174 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.ZkTestServer start zk server on port:40962
   [junit4]   2> 94201 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 94456 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 94523 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Exception causing close of session 0x0 due to java.io.IOException: ZooKeeperServer not running
   [junit4]   2> 95865 INFO  (zkCallback-41-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2e3034b5 name:ZooKeeperConnection Watcher:127.0.0.1:40962 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 95865 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 95865 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 95867 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 96424 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 96435 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 96471 INFO  (zkCallback-42-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@234d5125 name:ZooKeeperConnection Watcher:127.0.0.1:40962/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 96471 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 96472 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 96472 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1
   [junit4]   2> 96479 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/shards
   [junit4]   2> 96486 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection
   [junit4]   2> 96497 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection/shards
   [junit4]   2> 96504 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 96511 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 96835 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 96835 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/schema.xml
   [junit4]   2> 96849 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 96849 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 96863 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 96863 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 96872 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 96872 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/protwords.txt
   [junit4]   2> 96880 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 96880 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/currency.xml
   [junit4]   2> 96890 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 96890 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 96899 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 96899 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 96903 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 96904 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 96911 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 96911 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 96917 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 96918 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 96925 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 96939 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 97066 INFO  (zkCallback-43-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5109ce92 name:ZooKeeperConnection Watcher:127.0.0.1:40962 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 97071 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 97071 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 97071 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 97085 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 97091 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 97111 INFO  (zkCallback-44-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@308ccc60 name:ZooKeeperConnection Watcher:127.0.0.1:40962/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 97111 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 97111 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 97112 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf2/solrconfig.xml
   [junit4]   2> 97112 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/solrconfig.xml
   [junit4]   2> 97151 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf2/schema.xml
   [junit4]   2> 97151 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/schema.xml
   [junit4]   2> 97195 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf2/enumsConfig.xml
   [junit4]   2> 97195 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/enumsConfig.xml
   [junit4]   2> 97227 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf2/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 97227 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 97251 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf2/stopwords.txt
   [junit4]   2> 97251 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/stopwords.txt
   [junit4]   2> 97307 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf2/protwords.txt
   [junit4]   2> 97307 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/protwords.txt
   [junit4]   2> 97343 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf2/currency.xml
   [junit4]   2> 97343 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/currency.xml
   [junit4]   2> 97379 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf2/open-exchange-rates.json
   [junit4]   2> 97379 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/open-exchange-rates.json
   [junit4]   2> 97427 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf2/mapping-ISOLatin1Accent.txt
   [junit4]   2> 97427 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/mapping-ISOLatin1Accent.txt
   [junit4]   2> 97519 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf2/old_synonyms.txt
   [junit4]   2> 97519 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/old_synonyms.txt
   [junit4]   2> 97575 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf2/synonyms.txt
   [junit4]   2> 97575 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/synonyms.txt
   [junit4]   2> 97627 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/elevate.xml to /configs/conf2/elevate.xml
   [junit4]   2> 97627 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/elevate.xml
   [junit4]   2> 97800 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_327C025A1D389D1D-001/control-001/cores/collection1
   [junit4]   2> 97809 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 97814 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@639d645b{/,null,AVAILABLE}
   [junit4]   2> 97831 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.e.j.s.ServerConnector Started ServerConnector@7f09d68f{HTTP/1.1,[http/1.1]}{127.0.0.1:52467}
   [junit4]   2> 97831 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.e.j.s.Server Started @105059ms
   [junit4]   2> 97831 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_327C025A1D389D1D-001/tempDir-001/control/data, hostContext=/, hostPort=52467, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_327C025A1D389D1D-001/control-001/cores}
   [junit4]   2> 97831 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 97831 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_327C025A1D389D1D-001/control-001'
   [junit4]   2> 97831 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 97831 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 97832 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 97851 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 97887 INFO  (zkCallback-45-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@71354b85 name:ZooKeeperConnection Watcher:127.0.0.1:40962/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 97887 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 97887 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 97899 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 97899 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_327C025A1D389D1D-001/control-001/solr.xml
   [junit4]   2> 97918 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_327C025A1D389D1D-001/control-001/cores
   [junit4]   2> 97918 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.CoreContainer New CoreContainer 1565325419
   [junit4]   2> 97918 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_327C025A1D389D1D-001/control-001]
   [junit4]   2> 97919 WARN  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.CoreContainer Couldn't add files from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_327C025A1D389D1D-001/control-001/lib to classpath: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_327C025A1D389D1D-001/control-001/lib
   [junit4]   2> 97919 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,
   [junit4]   2> 97929 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.u.UpdateShardHandler Created UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&maxConnections=100000&maxConnectionsPerHost=100000
   [junit4]   2> 97929 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 97929 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 97929 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40962/solr
   [junit4]   2> 98129 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 98129 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 98130 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 98135 INFO  (zkCallback-48-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@1f7537 name:ZooKeeperConnection Watcher:127.0.0.1:40962 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 98135 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 98136 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 98150 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$108/20832659@798a2508
   [junit4]   2> 98155 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 98175 INFO  (zkCallback-49-thread-1-processing-n:127.0.0.1:52467_) [n:127.0.0.1:52467_    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6a22a12e name:ZooKeeperConnection Watcher:127.0.0.1:40962/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 98175 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 98199 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
   [junit4]   2> 98228 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
   [junit4]   2> 98247 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
   [junit4]   2> 98259 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
   [junit4]   2> 98262 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
   [junit4]   2> 98284 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 98287 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 98289 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 98291 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 98293 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 98304 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 98314 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 98366 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
   [junit4]   2> 98368 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
   [junit4]   2> 98371 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 98374 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/96560126554013702-127.0.0.1:52467_-n_0000000000
   [junit4]   2> 98378 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:52467_
   [junit4]   2> 98378 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
   [junit4]   2> 98431 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.Overseer Overseer (id=96560126554013702-127.0.0.1:52467_-n_0000000000) starting
   [junit4]   2> 98503 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
   [junit4]   2> 98790 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=10000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 98800 INFO  (OverseerCollectionConfigSetProcessor-96560126554013702-127.0.0.1:52467_-n_0000000000) [n:127.0.0.1:52467_    ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
   [junit4]   2> 98799 INFO  (OverseerStateUpdate-96560126554013702-127.0.0.1:52467_-n_0000000000) [n:127.0.0.1:52467_    ] o.a.s.c.Overseer Starting to work on the main queue
   [junit4]   2> 98808 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52467_
   [junit4]   2> 98808 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:52467_
   [junit4]   2> 98827 INFO  (zkCallback-49-thread-1-processing-n:127.0.0.1:52467_) [n:127.0.0.1:52467_    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 98826 INFO  (OverseerStateUpdate-96560126554013702-127.0.0.1:52467_-n_0000000000) [n:127.0.0.1:52467_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 98841 INFO  (zkCallback-49-thread-1-processing-n:127.0.0.1:52467_) [n:127.0.0.1:52467_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (1)
   [junit4]   2> 99096 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 99096 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 99102 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_327C025A1D389D1D-001/control-001/cores
   [junit4]   2> 99115 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.

[...truncated too long message...]

lassicSimilarity, locale=sr-Latn-RS, timezone=America/Virgin
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=228931416,total=522190848
   [junit4]   2> NOTE: All tests run in this JVM: [TestObjectReleaseTracker, StatsComponentTest, MultiThreadedOCPTest, PluginInfoTest, TestMergePolicyConfig, StatelessScriptUpdateProcessorFactoryTest, TestGroupingSearch, SolrCoreCheckLockOnStartupTest, TestSolrXml, TestPostingsSolrHighlighter, SuggesterTSTTest, TestManagedSynonymFilterFactory, TestFreeTextSuggestions, DistributedFacetPivotSmallTest, FullSolrCloudDistribCmdsTest, TestAnalyzeInfixSuggestions, LukeRequestHandlerTest, TestDistributedMissingSort, NoCacheHeaderTest, DeleteStatusTest, TestDistribIDF, CircularListTest, AutoCommitTest, TestLFUCache, SuggesterTest, TestSolrQueryParserResource, BlockJoinFacetDistribTest, AddSchemaFieldsUpdateProcessorFactoryTest, SolrPluginUtilsTest, GraphQueryTest, DirectUpdateHandlerTest, CloneFieldUpdateProcessorFactoryTest, TolerantUpdateProcessorTest, TestSubQueryTransformerCrossCore, QueryParsingTest, DistribJoinFromCollectionTest, FieldAnalysisRequestHandlerTest, BlockDirectoryTest, TestSolrDeletionPolicy1, TestSchemaManager, ZkStateReaderTest, TestUninvertingReader, TestFieldResource, TestFilteredDocIdSet, TestExceedMaxTermLength, TestXIncludeConfig, SpatialHeatmapFacetsTest, EnumFieldTest, TestExtendedDismaxParser, DistributedQueryComponentOptimizationTest, TestSort, JSONWriterTest, TestPhraseSuggestions, InfoHandlerTest, TestComplexPhraseQParserPlugin, TestStressUserVersions, TestTestInjection, CoreAdminHandlerTest, BasicDistributedZk2Test, DocValuesMultiTest, TestOnReconnectListenerSupport, DeleteNodeTest, SharedFSAutoReplicaFailoverTest, CustomCollectionTest, TestLegacyFieldCache, TestCloudManagedSchema, TestTolerantUpdateProcessorCloud, TestConfigSetImmutable, UnloadDistributedZkTest, HdfsCollectionsAPIDistributedZkTest, CoreAdminCreateDiscoverTest, TimeZoneUtilsTest, DOMUtilTest, TestSolrCloudWithSecureImpersonation, HdfsThreadLeakTest, FieldMutatingUpdateProcessorTest, TestSlowCompositeReaderWrapper, TestSearcherReuse, HdfsWriteToMultipleCollectionsTest, ConfigSetsAPITest, RuleEngineTest, TestPerFieldSimilarityWithDefaultOverride, SSLMigrationTest, OpenExchangeRatesOrgProviderTest, SuggesterWFSTTest, SolrCLIZkUtilsTest, PrimitiveFieldTypeTest, TestIntervalFaceting, BitVectorTest, ChangedSchemaMergeTest, ResponseLogComponentTest, TestDynamicFieldCollectionResource, TestRequestStatusCollectionAPI, HdfsLockFactoryTest, TestZkChroot, TestQueryWrapperFilter, TestSchemaVersionResource, DirectoryFactoryTest, TestStressReorder, TestFieldCacheReopen, BadIndexSchemaTest, TestClassicSimilarityFactory, ChaosMonkeySafeLeaderTest, SyncSliceTest, OpenCloseCoreStressTest, OverseerTest, LeaderElectionIntegrationTest, ShardRoutingTest, RecoveryZkTest, TestRandomFaceting, LeaderElectionTest, ZkSolrClientTest, TestDistributedSearch, PreAnalyzedFieldTest, TestSystemIdResolver, PrimUtilsTest, RAMDirectoryFactoryTest, TestSolrJ, TestLRUCache, TestUtils, ZkNodePropsTest, SliceStateTest, FileUtilsTest, TestRTGBase, CursorPagingTest, DistributedIntervalFacetingTest, AsyncMigrateRouteKeyTest, CleanupOldIndexTest, CloudExitableDirectoryReaderTest, CollectionReloadTest, CollectionStateFormat2Test, CollectionsAPIAsyncDistributedZkTest, ConcurrentDeleteAndCreateCollectionTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ConcurrentDeleteAndCreateCollectionTest -Dtests.seed=327C025A1D389D1D -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=sr-Latn-RS -Dtests.timezone=America/Virgin -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J2 | ConcurrentDeleteAndCreateCollectionTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([327C025A1D389D1D]:0)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 
   [junit4]    >    1) Thread[id=73355, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    2) Thread[id=73350, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    3) Thread[id=73340, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    4) Thread[id=73330, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    5) Thread[id=73335, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    6) Thread[id=73320, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    7) Thread[id=73325, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    8) Thread[id=73315, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    9) Thread[id=73345, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   10) Thread[id=73360, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([327C025A1D389D1D]:0)
   [junit4] Completed [452/631 (7!)] on J2 in 88.80s, 2 tests, 1 failure, 1 error <<< FAILURES!

[...truncated 625 lines...]
   [junit4] JVM J2: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20160909_030821_326.sysout
   [junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid24515.hprof ...
   [junit4] Heap dump file created [487528590 bytes in 2.761 secs]
   [junit4] <<< JVM J2: EOF ----

[...truncated 11040 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:715: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid24515.hprof

Total time: 400 minutes 4 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




Re: [JENKINS] Lucene-Solr-NightlyTests-master - Build # 1111 - Still Failing

Posted by Steve Rowe <sa...@gmail.com>.
Looks like the lucene Jenkins slave needs an increase in the open files limit.

Here’s what I added to /etc/sysctl.conf on my Debian box to do this:

   fs.file-max = 2097152

Uwe, you coordinating the recent puppetization of the lucene slave - should we open an INFRA JIRA to make this change?

--
Steve
www.lucidworks.com

> On Sep 7, 2016, at 6:12 AM, Apache Jenkins Server <je...@builds.apache.org> wrote:
> 
> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1111/
> 
> 10 tests failed.
> FAILED:  org.apache.lucene.index.TestIndexWriterReader.testDuringAddDelete
> 
> Error Message:
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J2/temp/lucene.index.TestIndexWriterReader_61A624ED3323A1DA-001/index-SimpleFSDirectory-001/_24x_Lucene50_1.tim: Too many open files
> 
> Stack Trace:
> java.nio.file.FileSystemException: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J2/temp/lucene.index.TestIndexWriterReader_61A624ED3323A1DA-001/index-SimpleFSDirectory-001/_24x_Lucene50_1.tim: Too many open files
> 	at __randomizedtesting.SeedInfo.seed([61A624ED3323A1DA:FF37536A19E223DA]:0)
> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> 	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:271)
> 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newByteChannel(FilterFileSystemProvider.java:212)
> 	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:240)
> 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newByteChannel(FilterFileSystemProvider.java:212)
> 	at java.nio.file.Files.newByteChannel(Files.java:361)
> 	at java.nio.file.Files.newByteChannel(Files.java:407)
> 	at org.apache.lucene.store.SimpleFSDirectory.openInput(SimpleFSDirectory.java:77)
> 	at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:99)
> 	at org.apache.lucene.codecs.blocktree.BlockTreeTermsReader.<init>(BlockTreeTermsReader.java:153)
> 	at org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.fieldsProducer(Lucene50PostingsFormat.java:445)
> 	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:261)
> 	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:341)
> 	at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:106)
> 	at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:74)
> 	at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:145)
> 	at org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:197)
> 	at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:103)
> 	at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:460)
> 	at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:291)
> 	at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:266)
> 	at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:256)
> 	at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:140)
> 	at org.apache.lucene.index.TestIndexWriterReader.testDuringAddDelete(TestIndexWriterReader.java:870)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:498)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> 
> FAILED:  junit.framework.TestSuite.org.apache.lucene.index.TestIndexWriterReader
> 
> Error Message:
> 5 threads leaked from SUITE scope at org.apache.lucene.index.TestIndexWriterReader:     1) Thread[id=1799, name=Thread-1549, state=BLOCKED, group=TGRP-TestIndexWriterReader]         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)    2) Thread[id=1801, name=Thread-1551, state=RUNNABLE, group=TGRP-TestIndexWriterReader]         at org.apache.lucene.codecs.compressing.CompressingTermVectorsReader.get(CompressingTermVectorsReader.java:417)         at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat$AssertingTermVectorsReader.get(AssertingTermVectorsFormat.java:71)         at org.apache.lucene.codecs.TermVectorsWriter.merge(TermVectorsWriter.java:223)         at org.apache.lucene.index.SegmentMerger.mergeVectors(SegmentMerger.java:210)         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:144)         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4287)         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3864)         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)    3) Thread[id=1802, name=Thread-1552, state=BLOCKED, group=TGRP-TestIndexWriterReader]         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)    4) Thread[id=1800, name=Thread-1550, state=BLOCKED, group=TGRP-TestIndexWriterReader]         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)    5) Thread[id=1798, name=Thread-1548, state=BLOCKED, group=TGRP-TestIndexWriterReader]         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
> 
> Stack Trace:
> com.carrotsearch.randomizedtesting.ThreadLeakError: 5 threads leaked from SUITE scope at org.apache.lucene.index.TestIndexWriterReader: 
>   1) Thread[id=1799, name=Thread-1549, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>        at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
>        at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>        at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>        at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>        at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   2) Thread[id=1801, name=Thread-1551, state=RUNNABLE, group=TGRP-TestIndexWriterReader]
>        at org.apache.lucene.codecs.compressing.CompressingTermVectorsReader.get(CompressingTermVectorsReader.java:417)
>        at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat$AssertingTermVectorsReader.get(AssertingTermVectorsFormat.java:71)
>        at org.apache.lucene.codecs.TermVectorsWriter.merge(TermVectorsWriter.java:223)
>        at org.apache.lucene.index.SegmentMerger.mergeVectors(SegmentMerger.java:210)
>        at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:144)
>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4287)
>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3864)
>        at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
>        at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>        at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>        at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>        at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   3) Thread[id=1802, name=Thread-1552, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>        at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
>        at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>        at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>        at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>        at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   4) Thread[id=1800, name=Thread-1550, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>        at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
>        at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>        at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>        at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>        at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   5) Thread[id=1798, name=Thread-1548, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>        at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
>        at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>        at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>        at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>        at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
> 	at __randomizedtesting.SeedInfo.seed([61A624ED3323A1DA]:0)
> 
> 
> FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIDistributedZkTest
> 
> Error Message:
> 1 thread leaked from SUITE scope at org.apache.solr.cloud.CollectionsAPIDistributedZkTest:     1) Thread[id=125227, name=searcherExecutor-10822-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)
> 
> Stack Trace:
> com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.CollectionsAPIDistributedZkTest: 
>   1) Thread[id=125227, name=searcherExecutor-10822-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
>        at sun.misc.Unsafe.park(Native Method)
>        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
>        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
>        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
>        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
>        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
>        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>        at java.lang.Thread.run(Thread.java:745)
> 	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32]:0)
> 
> 
> FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIDistributedZkTest
> 
> Error Message:
> There are still zombie threads that couldn't be terminated:    1) Thread[id=125227, name=searcherExecutor-10822-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)
> 
> Stack Trace:
> com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
>   1) Thread[id=125227, name=searcherExecutor-10822-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
>        at sun.misc.Unsafe.park(Native Method)
>        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
>        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
>        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
>        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
>        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
>        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>        at java.lang.Thread.run(Thread.java:745)
> 	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32]:0)
> 
> 
> FAILED:  org.apache.solr.cloud.hdfs.StressHdfsTest.test
> 
> Error Message:
> Could not find collection:delete_data_dir
> 
> Stack Trace:
> java.lang.AssertionError: Could not find collection:delete_data_dir
> 	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32:EE8AB604096653CA]:0)
> 	at org.junit.Assert.fail(Assert.java:93)
> 	at org.junit.Assert.assertTrue(Assert.java:43)
> 	at org.junit.Assert.assertNotNull(Assert.java:526)
> 	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:154)
> 	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:139)
> 	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:134)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:856)
> 	at org.apache.solr.cloud.hdfs.StressHdfsTest.createAndDeleteCollection(StressHdfsTest.java:159)
> 	at org.apache.solr.cloud.hdfs.StressHdfsTest.test(StressHdfsTest.java:103)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:498)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
> 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
> 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> 
> FAILED:  org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.test
> 
> Error Message:
> Timeout occured while waiting response from server at: https://127.0.0.1:39500/vum
> 
> Stack Trace:
> org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:39500/vum
> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
> 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
> 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:516)
> 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:498)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
> 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
> 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
> 	at java.lang.Thread.run(Thread.java:745)
> Caused by: java.net.SocketTimeoutException: Read timed out
> 	at java.net.SocketInputStream.socketRead0(Native Method)
> 	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
> 	at java.net.SocketInputStream.read(SocketInputStream.java:170)
> 	at java.net.SocketInputStream.read(SocketInputStream.java:141)
> 	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
> 	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
> 	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
> 	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
> 	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
> 	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
> 	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
> 	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
> 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
> 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
> 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
> 	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
> 	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
> 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
> 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
> 	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
> 	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
> 	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
> 	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
> 	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
> 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
> 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
> 	... 47 more
> 
> 
> FAILED:  org.apache.solr.handler.component.DistributedFacetPivotLongTailTest.test
> 
> Error Message:
> IOException occured when talking to server at: https://127.0.0.1:46729/_/d/collection1
> 
> Stack Trace:
> org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: https://127.0.0.1:46729/_/d/collection1
> 	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32:EE8AB604096653CA]:0)
> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:622)
> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
> 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
> 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
> 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
> 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:152)
> 	at org.apache.solr.handler.component.DistributedFacetPivotLongTailTest.test(DistributedFacetPivotLongTailTest.java:71)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:498)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
> 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
> 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
> 	at java.lang.Thread.run(Thread.java:745)
> Caused by: org.apache.http.conn.ConnectTimeoutException: Connect to 127.0.0.1:46729 [/127.0.0.1] failed: Read timed out
> 	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:143)
> 	at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
> 	at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
> 	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
> 	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
> 	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
> 	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
> 	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
> 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
> 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
> 	... 48 more
> Caused by: java.net.SocketTimeoutException: Read timed out
> 	at java.net.SocketInputStream.socketRead0(Native Method)
> 	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
> 	at java.net.SocketInputStream.read(SocketInputStream.java:170)
> 	at java.net.SocketInputStream.read(SocketInputStream.java:141)
> 	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
> 	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
> 	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
> 	at sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1375)
> 	at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1403)
> 	at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1387)
> 	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.createLayeredSocket(SSLConnectionSocketFactory.java:394)
> 	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:353)
> 	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:134)
> 	... 58 more
> 
> 
> FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test
> 
> Error Message:
> Timeout occured while waiting response from server at: https://127.0.0.1:35113
> 
> Stack Trace:
> org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:35113
> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
> 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
> 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:516)
> 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:498)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
> 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
> 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
> 	at java.lang.Thread.run(Thread.java:745)
> Caused by: java.net.SocketTimeoutException: Read timed out
> 	at java.net.SocketInputStream.socketRead0(Native Method)
> 	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
> 	at java.net.SocketInputStream.read(SocketInputStream.java:170)
> 	at java.net.SocketInputStream.read(SocketInputStream.java:141)
> 	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
> 	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
> 	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
> 	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
> 	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
> 	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
> 	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
> 	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
> 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
> 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
> 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
> 	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
> 	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
> 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
> 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
> 	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
> 	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
> 	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
> 	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
> 	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
> 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
> 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
> 	... 47 more
> 
> 
> FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest
> 
> Error Message:
> ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
> 
> Stack Trace:
> java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
> 	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32]:0)
> 	at org.junit.Assert.fail(Assert.java:93)
> 	at org.junit.Assert.assertTrue(Assert.java:43)
> 	at org.junit.Assert.assertNull(Assert.java:551)
> 	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
> 	at sun.reflect.GeneratedMethodAccessor73.invoke(Unknown Source)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:498)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> 
> FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest
> 
> Error Message:
> 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest:     1) Thread[id=28289, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=28294, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=28264, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=28259, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=28269, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=28304, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=28284, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=28299, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=28274, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=28279, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)
> 
> Stack Trace:
> com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 
>   1) Thread[id=28289, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
>        at java.lang.Thread.sleep(Native Method)
>        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
>        at java.lang.Thread.run(Thread.java:745)
>   2) Thread[id=28294, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
>        at java.lang.Thread.sleep(Native Method)
>        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
>        at java.lang.Thread.run(Thread.java:745)
>   3) Thread[id=28264, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
>        at java.lang.Thread.sleep(Native Method)
>        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
>        at java.lang.Thread.run(Thread.java:745)
>   4) Thread[id=28259, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
>        at java.lang.Thread.sleep(Native Method)
>        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
>        at java.lang.Thread.run(Thread.java:745)
>   5) Thread[id=28269, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
>        at java.lang.Thread.sleep(Native Method)
>        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
>        at java.lang.Thread.run(Thread.java:745)
>   6) Thread[id=28304, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
>        at java.lang.Thread.sleep(Native Method)
>        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
>        at java.lang.Thread.run(Thread.java:745)
>   7) Thread[id=28284, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
>        at java.lang.Thread.sleep(Native Method)
>        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
>        at java.lang.Thread.run(Thread.java:745)
>   8) Thread[id=28299, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
>        at java.lang.Thread.sleep(Native Method)
>        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
>        at java.lang.Thread.run(Thread.java:745)
>   9) Thread[id=28274, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
>        at java.lang.Thread.sleep(Native Method)
>        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
>        at java.lang.Thread.run(Thread.java:745)
>  10) Thread[id=28279, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
>        at java.lang.Thread.sleep(Native Method)
>        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
>        at java.lang.Thread.run(Thread.java:745)
> 	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32]:0)
> 
> 
> 
> 
> Build Log:
> [...truncated 913 lines...]
>   [junit4] Suite: org.apache.lucene.index.TestIndexWriterReader
>   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestIndexWriterReader -Dtests.method=testDuringAddDelete -Dtests.seed=61A624ED3323A1DA -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=mk -Dtests.timezone=Europe/Skopje -Dtests.asserts=true -Dtests.file.encoding=UTF-8
>   [junit4] ERROR    142s J2 | TestIndexWriterReader.testDuringAddDelete <<<
>   [junit4]    > Throwable #1: java.nio.file.FileSystemException: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J2/temp/lucene.index.TestIndexWriterReader_61A624ED3323A1DA-001/index-SimpleFSDirectory-001/_24x_Lucene50_1.tim: Too many open files
>   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([61A624ED3323A1DA:FF37536A19E223DA]:0)
>   [junit4]    > 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
>   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
>   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:271)
>   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newByteChannel(FilterFileSystemProvider.java:212)
>   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:240)
>   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newByteChannel(FilterFileSystemProvider.java:212)
>   [junit4]    > 	at java.nio.file.Files.newByteChannel(Files.java:361)
>   [junit4]    > 	at java.nio.file.Files.newByteChannel(Files.java:407)
>   [junit4]    > 	at org.apache.lucene.store.SimpleFSDirectory.openInput(SimpleFSDirectory.java:77)
>   [junit4]    > 	at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:99)
>   [junit4]    > 	at org.apache.lucene.codecs.blocktree.BlockTreeTermsReader.<init>(BlockTreeTermsReader.java:153)
>   [junit4]    > 	at org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.fieldsProducer(Lucene50PostingsFormat.java:445)
>   [junit4]    > 	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:261)
>   [junit4]    > 	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:341)
>   [junit4]    > 	at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:106)
>   [junit4]    > 	at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:74)
>   [junit4]    > 	at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:145)
>   [junit4]    > 	at org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:197)
>   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:103)
>   [junit4]    > 	at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:460)
>   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:291)
>   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:266)
>   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:256)
>   [junit4]    > 	at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:140)
>   [junit4]    > 	at org.apache.lucene.index.TestIndexWriterReader.testDuringAddDelete(TestIndexWriterReader.java:870)
>   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
>   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J2/temp/lucene.index.TestIndexWriterReader_61A624ED3323A1DA-001
>   [junit4]   2> Sep 07, 2016 4:42:35 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
>   [junit4]   2> WARNING: Will linger awaiting termination of 5 leaked thread(s).
>   [junit4]   2> Sep 07, 2016 4:42:55 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
>   [junit4]   2> SEVERE: 5 threads leaked from SUITE scope at org.apache.lucene.index.TestIndexWriterReader: 
>   [junit4]   2>    1) Thread[id=1799, name=Thread-1549, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   [junit4]   2>    2) Thread[id=1801, name=Thread-1551, state=RUNNABLE, group=TGRP-TestIndexWriterReader]
>   [junit4]   2>         at org.apache.lucene.codecs.compressing.CompressingTermVectorsReader.get(CompressingTermVectorsReader.java:417)
>   [junit4]   2>         at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat$AssertingTermVectorsReader.get(AssertingTermVectorsFormat.java:71)
>   [junit4]   2>         at org.apache.lucene.codecs.TermVectorsWriter.merge(TermVectorsWriter.java:223)
>   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.mergeVectors(SegmentMerger.java:210)
>   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:144)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4287)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3864)
>   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   [junit4]   2>    3) Thread[id=1802, name=Thread-1552, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   [junit4]   2>    4) Thread[id=1800, name=Thread-1550, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   [junit4]   2>    5) Thread[id=1798, name=Thread-1548, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   [junit4]   2> Sep 07, 2016 4:42:55 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
>   [junit4]   2> INFO: Starting to interrupt leaked threads:
>   [junit4]   2>    1) Thread[id=1799, name=Thread-1549, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>   [junit4]   2>    2) Thread[id=1801, name=Thread-1551, state=RUNNABLE, group=TGRP-TestIndexWriterReader]
>   [junit4]   2>    3) Thread[id=1802, name=Thread-1552, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>   [junit4]   2>    4) Thread[id=1800, name=Thread-1550, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>   [junit4]   2>    5) Thread[id=1798, name=Thread-1548, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>   [junit4]   2> Sep 07, 2016 4:42:55 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
>   [junit4]   2> INFO: All leaked threads terminated.
>   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {field1=Lucene50(blocksize=128), field=Lucene50(blocksize=128), indexname=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), foo=Lucene50(blocksize=128), id=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), field3=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), field2=FSTOrd50, field5=Lucene50(blocksize=128), field4=Lucene50(blocksize=128)}, docValues:{}, maxPointsInLeafNode=1314, maxMBSortInHeap=5.621346163809926, sim=RandomSimilarity(queryNorm=true): {field1=DFR I(F)B1, field=DFR I(n)L2, indexname=DFR GLZ(0.3), foo=DFR GLZ(0.3), field3=DFR I(ne)Z(0.3), field2=IB SPL-D1, field5=DFR I(F)1, field4=DFR I(F)LZ(0.3)}, locale=mk, timezone=Europe/Skopje
>   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=139897912,total=268959744
>   [junit4]   2> NOTE: All tests run in this JVM: [Test2BPagedBytes, TestStressIndexing, TestGeoUtils, TestNoMergePolicy, TestRollback, TestBlockPostingsFormat, TestTermQuery, TestTopDocsMerge, TestFilterIterator, TestStressDeletes, TestIntsRef, TestDocBoost, TestNamedSPILoader, TestTransactions, TestRollingBuffer, TestFilterWeight, TestIndexWriterThreadsToSegments, TestLogMergePolicy, TestHighCompressionMode, TestDocValuesIndexing, TestRegexpRandom2, TestSingleInstanceLockFactory, TestLucene62SegmentInfoFormat, TestSegmentMerger, TestIndexWriterCommit, TestNRTCachingDirectory, TestAllFilesHaveCodecHeader, TestFuzzyQuery, Test2BPostings, TestDocumentsWriterDeleteQueue, TestByteBlockPool, TestScorerPerf, TestPayloadsOnVectors, TestParallelLeafReader, TestSegmentTermDocs, TestParallelTermEnum, TestDocument, TestVersion, TestAtomicUpdate, TestConjunctionDISI, TestBagOfPostings, TestMultiTermConstantScore, TestSnapshotDeletionPolicy, TestMmapDirectory, TestSpansEnum, TestSpanNotQuery, TestTimSorter, TestWildcardRandom, TestInPlaceMergeSorter, TestFixedBitDocIdSet, TestSimilarity, TestSearchWithThreads, TestCharacterUtils, TestUpgradeIndexMergePolicy, TestBlendedTermQuery, TestBooleanQuery, TestArrayUtil, TestSimpleSearchEquivalence, TestSpanMultiTermQueryWrapper, TestDeletionPolicy, TestMultiPhraseEnum, TestLazyProxSkipping, TestForUtil, TestSpanExplanations, TestReaderWrapperDVTypeCheck, TestFastCompressionMode, TestRegexpQuery, TestTopFieldCollector, TestMSBRadixSorter, TestIndexWriterMerging, TestFileSwitchDirectory, TestIndexWriter, TestIndexWriterReader]
>   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestIndexWriterReader -Dtests.seed=61A624ED3323A1DA -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=mk -Dtests.timezone=Europe/Skopje -Dtests.asserts=true -Dtests.file.encoding=UTF-8
>   [junit4] ERROR   0.00s J2 | TestIndexWriterReader (suite) <<<
>   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 5 threads leaked from SUITE scope at org.apache.lucene.index.TestIndexWriterReader: 
>   [junit4]    >    1) Thread[id=1799, name=Thread-1549, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>   [junit4]    >         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>   [junit4]    >         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>   [junit4]    >         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   [junit4]    >    2) Thread[id=1801, name=Thread-1551, state=RUNNABLE, group=TGRP-TestIndexWriterReader]
>   [junit4]    >         at org.apache.lucene.codecs.compressing.CompressingTermVectorsReader.get(CompressingTermVectorsReader.java:417)
>   [junit4]    >         at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat$AssertingTermVectorsReader.get(AssertingTermVectorsFormat.java:71)
>   [junit4]    >         at org.apache.lucene.codecs.TermVectorsWriter.merge(TermVectorsWriter.java:223)
>   [junit4]    >         at org.apache.lucene.index.SegmentMerger.mergeVectors(SegmentMerger.java:210)
>   [junit4]    >         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:144)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4287)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3864)
>   [junit4]    >         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>   [junit4]    >         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>   [junit4]    >         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   [junit4]    >    3) Thread[id=1802, name=Thread-1552, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>   [junit4]    >         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>   [junit4]    >         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>   [junit4]    >         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   [junit4]    >    4) Thread[id=1800, name=Thread-1550, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>   [junit4]    >         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>   [junit4]    >         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>   [junit4]    >         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   [junit4]    >    5) Thread[id=1798, name=Thread-1548, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>   [junit4]    >         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>   [junit4]    >         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>   [junit4]    >         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([61A624ED3323A1DA]:0)
>   [junit4] Completed [200/432 (1!)] on J2 in 252.56s, 23 tests, 2 errors <<< FAILURES!
> 
> [...truncated 10658 lines...]
>   [junit4] Suite: org.apache.solr.cloud.hdfs.StressHdfsTest
>   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.StressHdfsTest_66DE89DEA79A3E32-001/init-core-data-001
>   [junit4]   2> 814899 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
>   [junit4]   2> 814899 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /_a/m
>   [junit4]   1> Formatting using clusterid: testClusterID
>   [junit4]   2> 815026 WARN  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
>   [junit4]   2> 815069 WARN  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
>   [junit4]   2> 815081 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log jetty-6.1.26
>   [junit4]   2> 815125 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_33702_hdfs____13ay83/webapp
>   [junit4]   2> 816226 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:33702
>   [junit4]   2> 817177 WARN  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
>   [junit4]   2> 817179 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log jetty-6.1.26
>   [junit4]   2> 817200 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_46024_datanode____.2mcfop/webapp
>   [junit4]   2> 818238 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:46024
>   [junit4]   2> 818469 WARN  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
>   [junit4]   2> 818470 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log jetty-6.1.26
>   [junit4]   2> 818502 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_55265_datanode____.v9bv0o/webapp
>   [junit4]   2> 818789 INFO  (IPC Server handler 6 on 59517) [    ] BlockStateChange BLOCK* processReport: from storage DS-124aedd7-53e1-4abe-9593-1749020619f8 node DatanodeRegistration(127.0.0.1:39806, datanodeUuid=44048ab0-2317-4266-b18a-947e4bc5cd29, infoPort=51796, infoSecurePort=0, ipcPort=39160, storageInfo=lv=-56;cid=testClusterID;nsid=2101821981;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
>   [junit4]   2> 818789 INFO  (IPC Server handler 6 on 59517) [    ] BlockStateChange BLOCK* processReport: from storage DS-0427d4bd-1eeb-4af5-a408-0b312310f3bb node DatanodeRegistration(127.0.0.1:39806, datanodeUuid=44048ab0-2317-4266-b18a-947e4bc5cd29, infoPort=51796, infoSecurePort=0, ipcPort=39160, storageInfo=lv=-56;cid=testClusterID;nsid=2101821981;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
>   [junit4]   2> 819366 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:55265
>   [junit4]   2> 819712 INFO  (IPC Server handler 3 on 59517) [    ] BlockStateChange BLOCK* processReport: from storage DS-0a413b87-a6df-47f1-b731-f3928bdb7bd0 node DatanodeRegistration(127.0.0.1:38537, datanodeUuid=6109fe6a-418b-4810-8965-110624cd978c, infoPort=48162, infoSecurePort=0, ipcPort=33212, storageInfo=lv=-56;cid=testClusterID;nsid=2101821981;c=0), blocks: 0, hasStaleStorage: true, processing time: 86 msecs
>   [junit4]   2> 819712 INFO  (IPC Server handler 3 on 59517) [    ] BlockStateChange BLOCK* processReport: from storage DS-394f120c-9225-45db-9d4d-a79e0357d0a0 node DatanodeRegistration(127.0.0.1:38537, datanodeUuid=6109fe6a-418b-4810-8965-110624cd978c, infoPort=48162, infoSecurePort=0, ipcPort=33212, storageInfo=lv=-56;cid=testClusterID;nsid=2101821981;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
>   [junit4]   2> 819976 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
>   [junit4]   2> 819977 INFO  (Thread-1303) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
>   [junit4]   2> 819977 INFO  (Thread-1303) [    ] o.a.s.c.ZkTestServer Starting server
>   [junit4]   2> 820180 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.ZkTestServer start zk server on port:43044
>   [junit4]   2> 820180 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
>   [junit4]   2> 820189 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
>   [junit4]   2> 820201 INFO  (zkCallback-698-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6ea7009c name:ZooKeeperConnection Watcher:127.0.0.1:43044 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 820201 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>   [junit4]   2> 820201 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
>   [junit4]   2> 820201 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
>   [junit4]   2> 820210 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
>   [junit4]   2> 820215 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
>   [junit4]   2> 820239 INFO  (zkCallback-699-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@1095833c name:ZooKeeperConnection Watcher:127.0.0.1:43044/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 820241 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>   [junit4]   2> 820241 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
>   [junit4]   2> 820241 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1
>   [junit4]   2> 820242 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/shards
>   [junit4]   2> 820244 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection
>   [junit4]   2> 820244 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection/shards
>   [junit4]   2> 820250 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
>   [junit4]   2> 820250 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.xml
>   [junit4]   2> 820252 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
>   [junit4]   2> 820252 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/schema.xml
>   [junit4]   2> 820253 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>   [junit4]   2> 820253 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>   [junit4]   2> 820255 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
>   [junit4]   2> 820255 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/stopwords.txt
>   [junit4]   2> 820262 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
>   [junit4]   2> 820262 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/protwords.txt
>   [junit4]   2> 820263 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
>   [junit4]   2> 820263 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/currency.xml
>   [junit4]   2> 820264 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
>   [junit4]   2> 820264 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/enumsConfig.xml
>   [junit4]   2> 820270 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenki
> 
> [...truncated too long message...]
> 
> ctory Closing directory, CoreContainer#isShutdown=true
>   [junit4]   2> 4696298 INFO  (coreCloseExecutor-5736-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory Closing directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedFacetPivotLongTailTest_66DE89DEA79A3E32-001/tempDir-001/shard2/cores/collection1/data/index
>   [junit4]   2> 4696305 INFO  (TEST-DistributedFacetPivotLongTailTest.test-seed#[66DE89DEA79A3E32]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@17e66fe8{/_/d,null,UNAVAILABLE}
>   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=DistributedFacetPivotLongTailTest -Dtests.method=test -Dtests.seed=66DE89DEA79A3E32 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=cs -Dtests.timezone=Etc/GMT-6 -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
>   [junit4] ERROR    131s J0 | DistributedFacetPivotLongTailTest.test <<<
>   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: https://127.0.0.1:46729/_/d/collection1
>   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32:EE8AB604096653CA]:0)
>   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:622)
>   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
>   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
>   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
>   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
>   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
>   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:152)
>   [junit4]    > 	at org.apache.solr.handler.component.DistributedFacetPivotLongTailTest.test(DistributedFacetPivotLongTailTest.java:71)
>   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
>   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
>   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
>   [junit4]    > Caused by: org.apache.http.conn.ConnectTimeoutException: Connect to 127.0.0.1:46729 [/127.0.0.1] failed: Read timed out
>   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:143)
>   [junit4]    > 	at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
>   [junit4]    > 	at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
>   [junit4]    > 	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
>   [junit4]    > 	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
>   [junit4]    > 	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
>   [junit4]    > 	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
>   [junit4]    > 	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
>   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
>   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
>   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
>   [junit4]    > 	... 48 more
>   [junit4]    > Caused by: java.net.SocketTimeoutException: Read timed out
>   [junit4]    > 	at java.net.SocketInputStream.socketRead0(Native Method)
>   [junit4]    > 	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
>   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:170)
>   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:141)
>   [junit4]    > 	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
>   [junit4]    > 	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
>   [junit4]    > 	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
>   [junit4]    > 	at sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1375)
>   [junit4]    > 	at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1403)
>   [junit4]    > 	at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1387)
>   [junit4]    > 	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.createLayeredSocket(SSLConnectionSocketFactory.java:394)
>   [junit4]    > 	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:353)
>   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:134)
>   [junit4]    > 	... 58 more
>   [junit4]   2> 4696316 INFO  (SUITE-DistributedFacetPivotLongTailTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
>   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedFacetPivotLongTailTest_66DE89DEA79A3E32-001
>   [junit4]   2> NOTE: test params are: codec=Lucene62, sim=ClassicSimilarity, locale=cs, timezone=Etc/GMT-6
>   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=251613816,total=477626368
>   [junit4]   2> NOTE: All tests run in this JVM: [OverseerModifyCollectionTest, RequiredFieldsTest, DistributedDebugComponentTest, TestMissingGroups, TestIBSimilarityFactory, CdcrBootstrapTest, TestManagedSynonymFilterFactory, LeaderInitiatedRecoveryOnShardRestartTest, TestSubQueryTransformerCrossCore, TestOnReconnectListenerSupport, ZkStateReaderTest, TestSolrQueryParserResource, ResourceLoaderTest, TestConfigSetImmutable, TestSolr4Spatial, TestFreeTextSuggestions, TestReplicationHandlerBackup, ReturnFieldsTest, TestClassicSimilarityFactory, HdfsThreadLeakTest, SharedFSAutoReplicaFailoverUtilsTest, TestCSVLoader, LeaderElectionIntegrationTest, ZkNodePropsTest, TestRemoteStreaming, TestSchemaManager, EnumFieldTest, BasicZkTest, DirectoryFactoryTest, RemoteQueryErrorTest, SOLR749Test, TestLMDirichletSimilarityFactory, TestStressUserVersions, TestFieldCacheSortRandom, TestDistribDocBasedVersion, ChangedSchemaMergeTest, RuleEngineTest, TestPhraseSuggestions, TestCloudPseudoReturnFields, TestReplicationHandler, DisMaxRequestHandlerTest, TestLegacyFieldCache, TestComplexPhraseQParserPlugin, BasicDistributedZkTest, TestLRUCache, TestDistributedSearch, RestartWhileUpdatingTest, TestMergePolicyConfig, TestRebalanceLeaders, TestInitParams, CacheHeaderTest, BitVectorTest, TestArbitraryIndexDir, TestCloudDeleteByQuery, PathHierarchyTokenizerFactoryTest, TestSchemaNameResource, TestOrdValues, DistributedFacetPivotLargeTest, BlockJoinFacetDistribTest, TestFaceting, SynonymTokenizerTest, SpatialHeatmapFacetsTest, TestMiniSolrCloudClusterBase, OutOfBoxZkACLAndCredentialsProvidersTest, QueryEqualityTest, IndexSchemaTest, TestBadConfig, TestAddFieldRealTimeGet, BlockDirectoryTest, DefaultValueUpdateProcessorTest, TestCollationFieldDocValues, TestFoldingMultitermQuery, TestBlobHandler, SolrCloudExampleTest, DistributedSpellCheckComponentTest, ConnectionReuseTest, PingRequestHandlerTest, DocumentAnalysisRequestHandlerTest, AddSchemaFieldsUpdateProcessorFactoryTest, HdfsCollectionsAPIDistributedZkTest, TestRTGBase, CloneFieldUpdateProcessorFactoryTest, TestSizeLimitedDistributedMap, TestLuceneMatchVersion, TestRawResponseWriter, SignatureUpdateProcessorFactoryTest, TestXmlQParserPlugin, TestComponentsName, TestSQLHandler, HdfsLockFactoryTest, TestIntervalFaceting, TestCloudManagedSchema, TestDynamicFieldCollectionResource, TestSortingResponseWriter, TestHighlightDedupGrouping, UUIDFieldTest, ConnectionManagerTest, AnalysisAfterCoreReloadTest, TestSchemaVersionResource, JSONWriterTest, TestUninvertingReader, TermVectorComponentDistributedTest, TestGroupingSearch, SimpleCollectionCreateDeleteTest, DistributedFacetPivotLongTailTest]
>   [junit4] Completed [632/633 (5!)] on J0 in 130.67s, 1 test, 1 error <<< FAILURES!
> 
> [...truncated 1 lines...]
>   [junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20160907_083928_132.sysout
>   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
>   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
>   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid30315.hprof ...
>   [junit4] Heap dump file created [707246725 bytes in 19.813 secs]
>   [junit4] <<< JVM J0: EOF ----
> 
> [...truncated 11039 lines...]
> BUILD FAILED
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:715: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
> * java_pid30315.hprof
> 
> Total time: 367 minutes 35 seconds
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> Recording test results
> Email was triggered for: Failure - Any
> Sending email for trigger: Failure - Any
> 
> 
> 
> 
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1111 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1111/

10 tests failed.
FAILED:  org.apache.lucene.index.TestIndexWriterReader.testDuringAddDelete

Error Message:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J2/temp/lucene.index.TestIndexWriterReader_61A624ED3323A1DA-001/index-SimpleFSDirectory-001/_24x_Lucene50_1.tim: Too many open files

Stack Trace:
java.nio.file.FileSystemException: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J2/temp/lucene.index.TestIndexWriterReader_61A624ED3323A1DA-001/index-SimpleFSDirectory-001/_24x_Lucene50_1.tim: Too many open files
	at __randomizedtesting.SeedInfo.seed([61A624ED3323A1DA:FF37536A19E223DA]:0)
	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:271)
	at org.apache.lucene.mockfile.FilterFileSystemProvider.newByteChannel(FilterFileSystemProvider.java:212)
	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:240)
	at org.apache.lucene.mockfile.FilterFileSystemProvider.newByteChannel(FilterFileSystemProvider.java:212)
	at java.nio.file.Files.newByteChannel(Files.java:361)
	at java.nio.file.Files.newByteChannel(Files.java:407)
	at org.apache.lucene.store.SimpleFSDirectory.openInput(SimpleFSDirectory.java:77)
	at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:99)
	at org.apache.lucene.codecs.blocktree.BlockTreeTermsReader.<init>(BlockTreeTermsReader.java:153)
	at org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.fieldsProducer(Lucene50PostingsFormat.java:445)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:261)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:341)
	at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:106)
	at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:74)
	at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:145)
	at org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:197)
	at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:103)
	at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:460)
	at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:291)
	at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:266)
	at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:256)
	at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:140)
	at org.apache.lucene.index.TestIndexWriterReader.testDuringAddDelete(TestIndexWriterReader.java:870)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.lucene.index.TestIndexWriterReader

Error Message:
5 threads leaked from SUITE scope at org.apache.lucene.index.TestIndexWriterReader:     1) Thread[id=1799, name=Thread-1549, state=BLOCKED, group=TGRP-TestIndexWriterReader]         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)    2) Thread[id=1801, name=Thread-1551, state=RUNNABLE, group=TGRP-TestIndexWriterReader]         at org.apache.lucene.codecs.compressing.CompressingTermVectorsReader.get(CompressingTermVectorsReader.java:417)         at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat$AssertingTermVectorsReader.get(AssertingTermVectorsFormat.java:71)         at org.apache.lucene.codecs.TermVectorsWriter.merge(TermVectorsWriter.java:223)         at org.apache.lucene.index.SegmentMerger.mergeVectors(SegmentMerger.java:210)         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:144)         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4287)         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3864)         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)    3) Thread[id=1802, name=Thread-1552, state=BLOCKED, group=TGRP-TestIndexWriterReader]         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)    4) Thread[id=1800, name=Thread-1550, state=BLOCKED, group=TGRP-TestIndexWriterReader]         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)    5) Thread[id=1798, name=Thread-1548, state=BLOCKED, group=TGRP-TestIndexWriterReader]         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 5 threads leaked from SUITE scope at org.apache.lucene.index.TestIndexWriterReader: 
   1) Thread[id=1799, name=Thread-1549, state=BLOCKED, group=TGRP-TestIndexWriterReader]
        at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
        at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
        at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
        at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
        at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   2) Thread[id=1801, name=Thread-1551, state=RUNNABLE, group=TGRP-TestIndexWriterReader]
        at org.apache.lucene.codecs.compressing.CompressingTermVectorsReader.get(CompressingTermVectorsReader.java:417)
        at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat$AssertingTermVectorsReader.get(AssertingTermVectorsFormat.java:71)
        at org.apache.lucene.codecs.TermVectorsWriter.merge(TermVectorsWriter.java:223)
        at org.apache.lucene.index.SegmentMerger.mergeVectors(SegmentMerger.java:210)
        at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:144)
        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4287)
        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3864)
        at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
        at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
        at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
        at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
        at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   3) Thread[id=1802, name=Thread-1552, state=BLOCKED, group=TGRP-TestIndexWriterReader]
        at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
        at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
        at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
        at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
        at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   4) Thread[id=1800, name=Thread-1550, state=BLOCKED, group=TGRP-TestIndexWriterReader]
        at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
        at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
        at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
        at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
        at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   5) Thread[id=1798, name=Thread-1548, state=BLOCKED, group=TGRP-TestIndexWriterReader]
        at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
        at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
        at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
        at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
        at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
	at __randomizedtesting.SeedInfo.seed([61A624ED3323A1DA]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIDistributedZkTest

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.CollectionsAPIDistributedZkTest:     1) Thread[id=125227, name=searcherExecutor-10822-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.CollectionsAPIDistributedZkTest: 
   1) Thread[id=125227, name=searcherExecutor-10822-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIDistributedZkTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=125227, name=searcherExecutor-10822-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=125227, name=searcherExecutor-10822-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32]:0)


FAILED:  org.apache.solr.cloud.hdfs.StressHdfsTest.test

Error Message:
Could not find collection:delete_data_dir

Stack Trace:
java.lang.AssertionError: Could not find collection:delete_data_dir
	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32:EE8AB604096653CA]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:154)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:139)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:134)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:856)
	at org.apache.solr.cloud.hdfs.StressHdfsTest.createAndDeleteCollection(StressHdfsTest.java:159)
	at org.apache.solr.cloud.hdfs.StressHdfsTest.test(StressHdfsTest.java:103)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:39500/vum

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:39500/vum
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:516)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  org.apache.solr.handler.component.DistributedFacetPivotLongTailTest.test

Error Message:
IOException occured when talking to server at: https://127.0.0.1:46729/_/d/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: https://127.0.0.1:46729/_/d/collection1
	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32:EE8AB604096653CA]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:622)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:152)
	at org.apache.solr.handler.component.DistributedFacetPivotLongTailTest.test(DistributedFacetPivotLongTailTest.java:71)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.conn.ConnectTimeoutException: Connect to 127.0.0.1:46729 [/127.0.0.1] failed: Read timed out
	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:143)
	at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
	at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 48 more
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1375)
	at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1403)
	at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1387)
	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.createLayeredSocket(SSLConnectionSocketFactory.java:394)
	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:353)
	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:134)
	... 58 more


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:35113

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:35113
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:516)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.GeneratedMethodAccessor73.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest:     1) Thread[id=28289, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=28294, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=28264, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=28259, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=28269, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=28304, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=28284, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=28299, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=28274, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=28279, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 
   1) Thread[id=28289, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=28294, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=28264, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=28259, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=28269, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=28304, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=28284, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=28299, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=28274, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=28279, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32]:0)




Build Log:
[...truncated 913 lines...]
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterReader
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestIndexWriterReader -Dtests.method=testDuringAddDelete -Dtests.seed=61A624ED3323A1DA -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=mk -Dtests.timezone=Europe/Skopje -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR    142s J2 | TestIndexWriterReader.testDuringAddDelete <<<
   [junit4]    > Throwable #1: java.nio.file.FileSystemException: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J2/temp/lucene.index.TestIndexWriterReader_61A624ED3323A1DA-001/index-SimpleFSDirectory-001/_24x_Lucene50_1.tim: Too many open files
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([61A624ED3323A1DA:FF37536A19E223DA]:0)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:271)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newByteChannel(FilterFileSystemProvider.java:212)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:240)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newByteChannel(FilterFileSystemProvider.java:212)
   [junit4]    > 	at java.nio.file.Files.newByteChannel(Files.java:361)
   [junit4]    > 	at java.nio.file.Files.newByteChannel(Files.java:407)
   [junit4]    > 	at org.apache.lucene.store.SimpleFSDirectory.openInput(SimpleFSDirectory.java:77)
   [junit4]    > 	at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:99)
   [junit4]    > 	at org.apache.lucene.codecs.blocktree.BlockTreeTermsReader.<init>(BlockTreeTermsReader.java:153)
   [junit4]    > 	at org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.fieldsProducer(Lucene50PostingsFormat.java:445)
   [junit4]    > 	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:261)
   [junit4]    > 	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:341)
   [junit4]    > 	at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:106)
   [junit4]    > 	at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:74)
   [junit4]    > 	at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:145)
   [junit4]    > 	at org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:197)
   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:103)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:460)
   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:291)
   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:266)
   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:256)
   [junit4]    > 	at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:140)
   [junit4]    > 	at org.apache.lucene.index.TestIndexWriterReader.testDuringAddDelete(TestIndexWriterReader.java:870)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J2/temp/lucene.index.TestIndexWriterReader_61A624ED3323A1DA-001
   [junit4]   2> Sep 07, 2016 4:42:35 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 5 leaked thread(s).
   [junit4]   2> Sep 07, 2016 4:42:55 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 5 threads leaked from SUITE scope at org.apache.lucene.index.TestIndexWriterReader: 
   [junit4]   2>    1) Thread[id=1799, name=Thread-1549, state=BLOCKED, group=TGRP-TestIndexWriterReader]
   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   [junit4]   2>    2) Thread[id=1801, name=Thread-1551, state=RUNNABLE, group=TGRP-TestIndexWriterReader]
   [junit4]   2>         at org.apache.lucene.codecs.compressing.CompressingTermVectorsReader.get(CompressingTermVectorsReader.java:417)
   [junit4]   2>         at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat$AssertingTermVectorsReader.get(AssertingTermVectorsFormat.java:71)
   [junit4]   2>         at org.apache.lucene.codecs.TermVectorsWriter.merge(TermVectorsWriter.java:223)
   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.mergeVectors(SegmentMerger.java:210)
   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:144)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4287)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3864)
   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   [junit4]   2>    3) Thread[id=1802, name=Thread-1552, state=BLOCKED, group=TGRP-TestIndexWriterReader]
   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   [junit4]   2>    4) Thread[id=1800, name=Thread-1550, state=BLOCKED, group=TGRP-TestIndexWriterReader]
   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   [junit4]   2>    5) Thread[id=1798, name=Thread-1548, state=BLOCKED, group=TGRP-TestIndexWriterReader]
   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   [junit4]   2> Sep 07, 2016 4:42:55 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=1799, name=Thread-1549, state=BLOCKED, group=TGRP-TestIndexWriterReader]
   [junit4]   2>    2) Thread[id=1801, name=Thread-1551, state=RUNNABLE, group=TGRP-TestIndexWriterReader]
   [junit4]   2>    3) Thread[id=1802, name=Thread-1552, state=BLOCKED, group=TGRP-TestIndexWriterReader]
   [junit4]   2>    4) Thread[id=1800, name=Thread-1550, state=BLOCKED, group=TGRP-TestIndexWriterReader]
   [junit4]   2>    5) Thread[id=1798, name=Thread-1548, state=BLOCKED, group=TGRP-TestIndexWriterReader]
   [junit4]   2> Sep 07, 2016 4:42:55 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {field1=Lucene50(blocksize=128), field=Lucene50(blocksize=128), indexname=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), foo=Lucene50(blocksize=128), id=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), field3=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), field2=FSTOrd50, field5=Lucene50(blocksize=128), field4=Lucene50(blocksize=128)}, docValues:{}, maxPointsInLeafNode=1314, maxMBSortInHeap=5.621346163809926, sim=RandomSimilarity(queryNorm=true): {field1=DFR I(F)B1, field=DFR I(n)L2, indexname=DFR GLZ(0.3), foo=DFR GLZ(0.3), field3=DFR I(ne)Z(0.3), field2=IB SPL-D1, field5=DFR I(F)1, field4=DFR I(F)LZ(0.3)}, locale=mk, timezone=Europe/Skopje
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=139897912,total=268959744
   [junit4]   2> NOTE: All tests run in this JVM: [Test2BPagedBytes, TestStressIndexing, TestGeoUtils, TestNoMergePolicy, TestRollback, TestBlockPostingsFormat, TestTermQuery, TestTopDocsMerge, TestFilterIterator, TestStressDeletes, TestIntsRef, TestDocBoost, TestNamedSPILoader, TestTransactions, TestRollingBuffer, TestFilterWeight, TestIndexWriterThreadsToSegments, TestLogMergePolicy, TestHighCompressionMode, TestDocValuesIndexing, TestRegexpRandom2, TestSingleInstanceLockFactory, TestLucene62SegmentInfoFormat, TestSegmentMerger, TestIndexWriterCommit, TestNRTCachingDirectory, TestAllFilesHaveCodecHeader, TestFuzzyQuery, Test2BPostings, TestDocumentsWriterDeleteQueue, TestByteBlockPool, TestScorerPerf, TestPayloadsOnVectors, TestParallelLeafReader, TestSegmentTermDocs, TestParallelTermEnum, TestDocument, TestVersion, TestAtomicUpdate, TestConjunctionDISI, TestBagOfPostings, TestMultiTermConstantScore, TestSnapshotDeletionPolicy, TestMmapDirectory, TestSpansEnum, TestSpanNotQuery, TestTimSorter, TestWildcardRandom, TestInPlaceMergeSorter, TestFixedBitDocIdSet, TestSimilarity, TestSearchWithThreads, TestCharacterUtils, TestUpgradeIndexMergePolicy, TestBlendedTermQuery, TestBooleanQuery, TestArrayUtil, TestSimpleSearchEquivalence, TestSpanMultiTermQueryWrapper, TestDeletionPolicy, TestMultiPhraseEnum, TestLazyProxSkipping, TestForUtil, TestSpanExplanations, TestReaderWrapperDVTypeCheck, TestFastCompressionMode, TestRegexpQuery, TestTopFieldCollector, TestMSBRadixSorter, TestIndexWriterMerging, TestFileSwitchDirectory, TestIndexWriter, TestIndexWriterReader]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestIndexWriterReader -Dtests.seed=61A624ED3323A1DA -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=mk -Dtests.timezone=Europe/Skopje -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J2 | TestIndexWriterReader (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 5 threads leaked from SUITE scope at org.apache.lucene.index.TestIndexWriterReader: 
   [junit4]    >    1) Thread[id=1799, name=Thread-1549, state=BLOCKED, group=TGRP-TestIndexWriterReader]
   [junit4]    >         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
   [junit4]    >         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
   [junit4]    >         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   [junit4]    >    2) Thread[id=1801, name=Thread-1551, state=RUNNABLE, group=TGRP-TestIndexWriterReader]
   [junit4]    >         at org.apache.lucene.codecs.compressing.CompressingTermVectorsReader.get(CompressingTermVectorsReader.java:417)
   [junit4]    >         at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat$AssertingTermVectorsReader.get(AssertingTermVectorsFormat.java:71)
   [junit4]    >         at org.apache.lucene.codecs.TermVectorsWriter.merge(TermVectorsWriter.java:223)
   [junit4]    >         at org.apache.lucene.index.SegmentMerger.mergeVectors(SegmentMerger.java:210)
   [junit4]    >         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:144)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4287)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3864)
   [junit4]    >         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
   [junit4]    >         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
   [junit4]    >         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   [junit4]    >    3) Thread[id=1802, name=Thread-1552, state=BLOCKED, group=TGRP-TestIndexWriterReader]
   [junit4]    >         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
   [junit4]    >         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
   [junit4]    >         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   [junit4]    >    4) Thread[id=1800, name=Thread-1550, state=BLOCKED, group=TGRP-TestIndexWriterReader]
   [junit4]    >         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
   [junit4]    >         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
   [junit4]    >         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   [junit4]    >    5) Thread[id=1798, name=Thread-1548, state=BLOCKED, group=TGRP-TestIndexWriterReader]
   [junit4]    >         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
   [junit4]    >         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
   [junit4]    >         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([61A624ED3323A1DA]:0)
   [junit4] Completed [200/432 (1!)] on J2 in 252.56s, 23 tests, 2 errors <<< FAILURES!

[...truncated 10658 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.StressHdfsTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.StressHdfsTest_66DE89DEA79A3E32-001/init-core-data-001
   [junit4]   2> 814899 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 814899 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /_a/m
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 815026 WARN  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 815069 WARN  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 815081 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 815125 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_33702_hdfs____13ay83/webapp
   [junit4]   2> 816226 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:33702
   [junit4]   2> 817177 WARN  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 817179 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 817200 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_46024_datanode____.2mcfop/webapp
   [junit4]   2> 818238 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:46024
   [junit4]   2> 818469 WARN  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 818470 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 818502 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_55265_datanode____.v9bv0o/webapp
   [junit4]   2> 818789 INFO  (IPC Server handler 6 on 59517) [    ] BlockStateChange BLOCK* processReport: from storage DS-124aedd7-53e1-4abe-9593-1749020619f8 node DatanodeRegistration(127.0.0.1:39806, datanodeUuid=44048ab0-2317-4266-b18a-947e4bc5cd29, infoPort=51796, infoSecurePort=0, ipcPort=39160, storageInfo=lv=-56;cid=testClusterID;nsid=2101821981;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 818789 INFO  (IPC Server handler 6 on 59517) [    ] BlockStateChange BLOCK* processReport: from storage DS-0427d4bd-1eeb-4af5-a408-0b312310f3bb node DatanodeRegistration(127.0.0.1:39806, datanodeUuid=44048ab0-2317-4266-b18a-947e4bc5cd29, infoPort=51796, infoSecurePort=0, ipcPort=39160, storageInfo=lv=-56;cid=testClusterID;nsid=2101821981;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 819366 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:55265
   [junit4]   2> 819712 INFO  (IPC Server handler 3 on 59517) [    ] BlockStateChange BLOCK* processReport: from storage DS-0a413b87-a6df-47f1-b731-f3928bdb7bd0 node DatanodeRegistration(127.0.0.1:38537, datanodeUuid=6109fe6a-418b-4810-8965-110624cd978c, infoPort=48162, infoSecurePort=0, ipcPort=33212, storageInfo=lv=-56;cid=testClusterID;nsid=2101821981;c=0), blocks: 0, hasStaleStorage: true, processing time: 86 msecs
   [junit4]   2> 819712 INFO  (IPC Server handler 3 on 59517) [    ] BlockStateChange BLOCK* processReport: from storage DS-394f120c-9225-45db-9d4d-a79e0357d0a0 node DatanodeRegistration(127.0.0.1:38537, datanodeUuid=6109fe6a-418b-4810-8965-110624cd978c, infoPort=48162, infoSecurePort=0, ipcPort=33212, storageInfo=lv=-56;cid=testClusterID;nsid=2101821981;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 819976 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 819977 INFO  (Thread-1303) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 819977 INFO  (Thread-1303) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 820180 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.ZkTestServer start zk server on port:43044
   [junit4]   2> 820180 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 820189 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 820201 INFO  (zkCallback-698-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6ea7009c name:ZooKeeperConnection Watcher:127.0.0.1:43044 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 820201 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 820201 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 820201 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 820210 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 820215 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 820239 INFO  (zkCallback-699-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@1095833c name:ZooKeeperConnection Watcher:127.0.0.1:43044/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 820241 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 820241 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 820241 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1
   [junit4]   2> 820242 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/shards
   [junit4]   2> 820244 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection
   [junit4]   2> 820244 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection/shards
   [junit4]   2> 820250 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 820250 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 820252 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 820252 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/schema.xml
   [junit4]   2> 820253 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 820253 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 820255 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 820255 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 820262 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 820262 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/protwords.txt
   [junit4]   2> 820263 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 820263 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/currency.xml
   [junit4]   2> 820264 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 820264 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 820270 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenki

[...truncated too long message...]

ctory Closing directory, CoreContainer#isShutdown=true
   [junit4]   2> 4696298 INFO  (coreCloseExecutor-5736-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory Closing directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedFacetPivotLongTailTest_66DE89DEA79A3E32-001/tempDir-001/shard2/cores/collection1/data/index
   [junit4]   2> 4696305 INFO  (TEST-DistributedFacetPivotLongTailTest.test-seed#[66DE89DEA79A3E32]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@17e66fe8{/_/d,null,UNAVAILABLE}
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=DistributedFacetPivotLongTailTest -Dtests.method=test -Dtests.seed=66DE89DEA79A3E32 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=cs -Dtests.timezone=Etc/GMT-6 -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR    131s J0 | DistributedFacetPivotLongTailTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: https://127.0.0.1:46729/_/d/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32:EE8AB604096653CA]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:622)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:152)
   [junit4]    > 	at org.apache.solr.handler.component.DistributedFacetPivotLongTailTest.test(DistributedFacetPivotLongTailTest.java:71)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.conn.ConnectTimeoutException: Connect to 127.0.0.1:46729 [/127.0.0.1] failed: Read timed out
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
   [junit4]    > 	at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
   [junit4]    > 	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
   [junit4]    > 	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
   [junit4]    > 	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
   [junit4]    > 	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
   [junit4]    > 	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: java.net.SocketTimeoutException: Read timed out
   [junit4]    > 	at java.net.SocketInputStream.socketRead0(Native Method)
   [junit4]    > 	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:170)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:141)
   [junit4]    > 	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
   [junit4]    > 	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
   [junit4]    > 	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
   [junit4]    > 	at sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1375)
   [junit4]    > 	at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1403)
   [junit4]    > 	at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1387)
   [junit4]    > 	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.createLayeredSocket(SSLConnectionSocketFactory.java:394)
   [junit4]    > 	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:353)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:134)
   [junit4]    > 	... 58 more
   [junit4]   2> 4696316 INFO  (SUITE-DistributedFacetPivotLongTailTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedFacetPivotLongTailTest_66DE89DEA79A3E32-001
   [junit4]   2> NOTE: test params are: codec=Lucene62, sim=ClassicSimilarity, locale=cs, timezone=Etc/GMT-6
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=251613816,total=477626368
   [junit4]   2> NOTE: All tests run in this JVM: [OverseerModifyCollectionTest, RequiredFieldsTest, DistributedDebugComponentTest, TestMissingGroups, TestIBSimilarityFactory, CdcrBootstrapTest, TestManagedSynonymFilterFactory, LeaderInitiatedRecoveryOnShardRestartTest, TestSubQueryTransformerCrossCore, TestOnReconnectListenerSupport, ZkStateReaderTest, TestSolrQueryParserResource, ResourceLoaderTest, TestConfigSetImmutable, TestSolr4Spatial, TestFreeTextSuggestions, TestReplicationHandlerBackup, ReturnFieldsTest, TestClassicSimilarityFactory, HdfsThreadLeakTest, SharedFSAutoReplicaFailoverUtilsTest, TestCSVLoader, LeaderElectionIntegrationTest, ZkNodePropsTest, TestRemoteStreaming, TestSchemaManager, EnumFieldTest, BasicZkTest, DirectoryFactoryTest, RemoteQueryErrorTest, SOLR749Test, TestLMDirichletSimilarityFactory, TestStressUserVersions, TestFieldCacheSortRandom, TestDistribDocBasedVersion, ChangedSchemaMergeTest, RuleEngineTest, TestPhraseSuggestions, TestCloudPseudoReturnFields, TestReplicationHandler, DisMaxRequestHandlerTest, TestLegacyFieldCache, TestComplexPhraseQParserPlugin, BasicDistributedZkTest, TestLRUCache, TestDistributedSearch, RestartWhileUpdatingTest, TestMergePolicyConfig, TestRebalanceLeaders, TestInitParams, CacheHeaderTest, BitVectorTest, TestArbitraryIndexDir, TestCloudDeleteByQuery, PathHierarchyTokenizerFactoryTest, TestSchemaNameResource, TestOrdValues, DistributedFacetPivotLargeTest, BlockJoinFacetDistribTest, TestFaceting, SynonymTokenizerTest, SpatialHeatmapFacetsTest, TestMiniSolrCloudClusterBase, OutOfBoxZkACLAndCredentialsProvidersTest, QueryEqualityTest, IndexSchemaTest, TestBadConfig, TestAddFieldRealTimeGet, BlockDirectoryTest, DefaultValueUpdateProcessorTest, TestCollationFieldDocValues, TestFoldingMultitermQuery, TestBlobHandler, SolrCloudExampleTest, DistributedSpellCheckComponentTest, ConnectionReuseTest, PingRequestHandlerTest, DocumentAnalysisRequestHandlerTest, AddSchemaFieldsUpdateProcessorFactoryTest, HdfsCollectionsAPIDistributedZkTest, TestRTGBase, CloneFieldUpdateProcessorFactoryTest, TestSizeLimitedDistributedMap, TestLuceneMatchVersion, TestRawResponseWriter, SignatureUpdateProcessorFactoryTest, TestXmlQParserPlugin, TestComponentsName, TestSQLHandler, HdfsLockFactoryTest, TestIntervalFaceting, TestCloudManagedSchema, TestDynamicFieldCollectionResource, TestSortingResponseWriter, TestHighlightDedupGrouping, UUIDFieldTest, ConnectionManagerTest, AnalysisAfterCoreReloadTest, TestSchemaVersionResource, JSONWriterTest, TestUninvertingReader, TermVectorComponentDistributedTest, TestGroupingSearch, SimpleCollectionCreateDeleteTest, DistributedFacetPivotLongTailTest]
   [junit4] Completed [632/633 (5!)] on J0 in 130.67s, 1 test, 1 error <<< FAILURES!

[...truncated 1 lines...]
   [junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20160907_083928_132.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid30315.hprof ...
   [junit4] Heap dump file created [707246725 bytes in 19.813 secs]
   [junit4] <<< JVM J0: EOF ----

[...truncated 11039 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:715: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid30315.hprof

Total time: 367 minutes 35 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1110 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1110/

11 tests failed.
FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testUpdateLogSynchronisation

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard2

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard2
	at __randomizedtesting.SeedInfo.seed([C7E4E06A8290DA3E:398BB8C940B0F92F]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testUpdateLogSynchronisation(CdcrReplicationDistributedZkTest.java:377)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchAddsWithDelete

Error Message:
Timeout while trying to assert number of documents @ target_collection

Stack Trace:
java.lang.AssertionError: Timeout while trying to assert number of documents @ target_collection
	at __randomizedtesting.SeedInfo.seed([C7E4E06A8290DA3E:B0FE99CE2A89DB12]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:273)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchAddsWithDelete(CdcrReplicationDistributedZkTest.java:532)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.AssertionError: expected:<59> but was:<0>
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:264)
	... 42 more


FAILED:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=43351, name=Thread-27331, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=43351, name=Thread-27331, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:44445/yx/eg/collection1
	at __randomizedtesting.SeedInfo.seed([C7E4E06A8290DA3E]:0)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:644)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:44445/yx/eg/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:166)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:642)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 5 more


FAILED:  org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.test

Error Message:
Error from server at https://127.0.0.1:33378/h_ej/rc: KeeperErrorCode = NoNode for /overseer/collection-queue-work/qnr-0000000150

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:33378/h_ej/rc: KeeperErrorCode = NoNode for /overseer/collection-queue-work/qnr-0000000150
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:608)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1599)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1620)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.deleteCollectionWithDownNodes(CollectionsAPIDistributedZkTest.java:345)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:186)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.core.OpenCloseCoreStressTest.test10Minutes

Error Message:
Captured an uncaught exception in thread: Thread[id=22510, name=Thread-13374, state=RUNNABLE, group=TGRP-OpenCloseCoreStressTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=22510, name=Thread-13374, state=RUNNABLE, group=TGRP-OpenCloseCoreStressTest]
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
	at __randomizedtesting.SeedInfo.seed([C7E4E06A8290DA3E]:0)
	at sun.security.ssl.InputRecord.<init>(InputRecord.java:93)
	at sun.security.ssl.AppInputStream.<init>(AppInputStream.java:50)
	at sun.security.ssl.SSLSocketImpl.init(SSLSocketImpl.java:635)
	at sun.security.ssl.SSLSocketImpl.<init>(SSLSocketImpl.java:567)
	at sun.security.ssl.SSLSocketFactoryImpl.createSocket(SSLSocketFactoryImpl.java:110)
	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.createLayeredSocket(SSLConnectionSocketFactory.java:363)
	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:353)
	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:134)
	at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
	at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
	at org.apache.solr.core.OneQuery.run(OpenCloseCoreStressTest.java:502)


FAILED:  org.apache.solr.handler.component.DistributedFacetPivotLongTailTest.test

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded
	at __randomizedtesting.SeedInfo.seed([C7E4E06A8290DA3E:4FB0DFB02C6CB7C6]:0)
	at java.util.HashMap.resize(HashMap.java:703)
	at java.util.HashMap.putVal(HashMap.java:628)
	at java.util.HashMap.put(HashMap.java:611)
	at java.util.HashSet.add(HashSet.java:219)
	at java.util.AbstractCollection.addAll(AbstractCollection.java:344)
	at sun.security.ssl.SSLAlgorithmDecomposer.decompose(SSLAlgorithmDecomposer.java:212)
	at sun.security.ssl.SSLAlgorithmDecomposer.decompose(SSLAlgorithmDecomposer.java:243)
	at sun.security.util.AbstractAlgorithmConstraints.checkAlgorithm(AbstractAlgorithmConstraints.java:104)
	at sun.security.util.DisabledAlgorithmConstraints.permits(DisabledAlgorithmConstraints.java:91)
	at sun.security.ssl.SSLAlgorithmConstraints.permits(SSLAlgorithmConstraints.java:149)
	at sun.security.ssl.Handshaker.getActiveCipherSuites(Handshaker.java:642)
	at sun.security.ssl.Handshaker.activate(Handshaker.java:509)
	at sun.security.ssl.SSLSocketImpl.kickstartHandshake(SSLSocketImpl.java:1482)
	at sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1351)
	at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1403)
	at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1387)
	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.createLayeredSocket(SSLConnectionSocketFactory.java:394)
	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:353)
	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:134)
	at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
	at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)


FAILED:  org.apache.solr.security.BasicAuthIntegrationTest.testBasics

Error Message:
IOException occured when talking to server at: http://127.0.0.1:57585/solr/testSolrCloudCollection_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: IOException occured when talking to server at: http://127.0.0.1:57585/solr/testSolrCloudCollection_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([C7E4E06A8290DA3E:FA3C4E46BA7E844E]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:755)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1161)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1050)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:992)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.security.BasicAuthIntegrationTest.doExtraTests(BasicAuthIntegrationTest.java:193)
	at org.apache.solr.cloud.TestMiniSolrCloudClusterBase.testCollectionCreateSearchDelete(TestMiniSolrCloudClusterBase.java:196)
	at org.apache.solr.cloud.TestMiniSolrCloudClusterBase.testBasics(TestMiniSolrCloudClusterBase.java:79)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57585/solr/testSolrCloudCollection_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:622)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:415)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:367)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:730)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:57585 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 10 more


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchBoundaries

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard1

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard1
	at __randomizedtesting.SeedInfo.seed([C7E4E06A8290DA3E:E5CB393FFB341BD9]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchBoundaries(CdcrReplicationDistributedZkTest.java:557)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:50706/k_qp/b

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:50706/k_qp/b
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:477)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
	at __randomizedtesting.SeedInfo.seed([C7E4E06A8290DA3E]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest:     1) Thread[id=31388, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=31358, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=31368, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=31373, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=31393, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=31353, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=31348, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=31378, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=31363, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=31383, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 
   1) Thread[id=31388, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=31358, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=31368, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=31373, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=31393, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=31353, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=31348, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=31378, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=31363, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=31383, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([C7E4E06A8290DA3E]:0)




Build Log:
[...truncated 11527 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C7E4E06A8290DA3E-001/init-core-data-001
   [junit4]   2> 350938 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 350939 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /h_ej/rc
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 351035 WARN  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 351051 WARN  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 351058 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 351088 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_49283_hdfs____upt5te/webapp
   [junit4]   2> 351660 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:49283
   [junit4]   2> 351754 WARN  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 351755 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 351769 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_37236_datanode____.1na3xk/webapp
   [junit4]   2> 352484 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:37236
   [junit4]   2> 352831 WARN  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 352832 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 352861 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_48061_datanode____wmx0em/webapp
   [junit4]   2> 353096 INFO  (IPC Server handler 2 on 43110) [    ] BlockStateChange BLOCK* processReport: from storage DS-e2653e2a-8c7e-4133-818f-4bb20db57ef8 node DatanodeRegistration(127.0.0.1:40487, datanodeUuid=e4c267d3-903c-47ed-a021-0e4e34b9c543, infoPort=34091, infoSecurePort=0, ipcPort=53589, storageInfo=lv=-56;cid=testClusterID;nsid=1359521124;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 353096 INFO  (IPC Server handler 2 on 43110) [    ] BlockStateChange BLOCK* processReport: from storage DS-05280b5b-55ed-4af8-8cf4-55b99d1d8d26 node DatanodeRegistration(127.0.0.1:40487, datanodeUuid=e4c267d3-903c-47ed-a021-0e4e34b9c543, infoPort=34091, infoSecurePort=0, ipcPort=53589, storageInfo=lv=-56;cid=testClusterID;nsid=1359521124;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 353462 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:48061
   [junit4]   2> 353939 INFO  (IPC Server handler 4 on 43110) [    ] BlockStateChange BLOCK* processReport: from storage DS-5330f517-3544-4622-958e-bf2817ec78ae node DatanodeRegistration(127.0.0.1:60255, datanodeUuid=30b4148e-f09b-439d-b9f7-20fa3220b2b0, infoPort=54558, infoSecurePort=0, ipcPort=51718, storageInfo=lv=-56;cid=testClusterID;nsid=1359521124;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 353939 INFO  (IPC Server handler 4 on 43110) [    ] BlockStateChange BLOCK* processReport: from storage DS-44dc2c54-df24-4aa9-bee0-8edabac13b64 node DatanodeRegistration(127.0.0.1:60255, datanodeUuid=30b4148e-f09b-439d-b9f7-20fa3220b2b0, infoPort=54558, infoSecurePort=0, ipcPort=51718, storageInfo=lv=-56;cid=testClusterID;nsid=1359521124;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 354051 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 354054 INFO  (Thread-707) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 354054 INFO  (Thread-707) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 354154 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.ZkTestServer start zk server on port:39994
   [junit4]   2> 354154 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 354182 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 354190 INFO  (zkCallback-408-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@790a925b name:ZooKeeperConnection Watcher:127.0.0.1:39994 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 354191 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 354191 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 354191 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 354194 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 354199 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 354204 INFO  (zkCallback-409-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5662b9c0 name:ZooKeeperConnection Watcher:127.0.0.1:39994/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 354204 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 354205 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 354205 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1
   [junit4]   2> 354206 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/shards
   [junit4]   2> 354208 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection
   [junit4]   2> 354209 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection/shards
   [junit4]   2> 354210 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 354210 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 354213 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 354213 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/schema.xml
   [junit4]   2> 354214 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 354215 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 354216 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 354216 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 354218 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 354218 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/protwords.txt
   [junit4]   2> 354219 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 354219 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/currency.xml
   [junit4]   2> 354221 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 354221 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 354222 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 354222 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 354224 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 354224 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 354225 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 354225 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 354227 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 354227 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 354230 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 354230 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 354242 INFO  (zkCallback-411-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@592a7ee9 name:ZooKeeperConnection Watcher:127.0.0.1:39994/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 354242 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 354242 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 354244 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 354245 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 354252 INFO  (zkCallback-412-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2249b59d name:ZooKeeperConnection Watcher:127.0.0.1:39994 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 354253 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 354253 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 354253 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 354255 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 354255 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 354261 INFO  (zkCallback-413-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4925e88b name:ZooKeeperConnection Watcher:127.0.0.1:39994/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 354261 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 354261 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 354261 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf2/solrconfig.xml
   [junit4]   2> 354261 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/solrconfig.xml
   [junit4]   2> 354264 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf2/schema.xml
   [junit4]   2> 354264 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/schema.xml
   [junit4]   2> 354266 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf2/enumsConfig.xml
   [junit4]   2> 354266 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/enumsConfig.xml
   [junit4]   2> 354267 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf2/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 354267 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 354269 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf2/stopwords.txt
   [junit4]   2> 354269 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/stopwords.txt
   [junit4]   2> 354270 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf2/protwords.txt
   [junit4]   2> 354271 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/protwords.txt
   [junit4]   2> 354272 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf2/currency.xml
   [junit4]   2> 354272 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/currency.xml
   [junit4]   2> 354274 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf2/open-exchange-rates.json
   [junit4]   2> 354274 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/open-exchange-rates.json
   [junit4]   2> 354275 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf2/mapping-ISOLatin1Accent.txt
   [junit4]   2> 354275 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/mapping-ISOLatin1Accent.txt
   [junit4]   2> 354277 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf2/old_synonyms.txt
   [junit4]   2> 354277 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/old_synonyms.txt
   [junit4]   2> 354278 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf2/synonyms.txt
   [junit4]   2> 354278 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/synonyms.txt
   [junit4]   2> 354280 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/elevate.xml to /configs/conf2/elevate.xml
   [junit4]   2> 354280 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/elevate.xml
   [junit4]   2> 354384 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C7E4E06A8290DA3E-001/control-001/cores/collection1
   [junit4]   2> 354387 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 354388 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4840ebea{/h_ej/rc,null,AVAILABLE}
   [junit4]   2> 354390 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.e.j.s.ServerConnector Started ServerConnector@545dcd98{SSL,[ssl, http/1.1]}{127.0.0.1:59258}
   [junit4]   2> 354390 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.e.j.s.Server Started @358914ms
   [junit4]   2> 354390 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:43110/hdfs__localhost_43110__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-master_checkout_solr_build_solr-core_test_J0_temp_solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C7E4E06A8290DA3E-001_tempDir-002_control_data, hostContext=/h_ej/rc, hostPort=59258, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C7E4E06A8290DA3E-001/control-001/cores}
   [junit4]   2> 354391 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 354391 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C7E4E06A8290DA3E-001/control-001'
   [junit4]   2> 354391 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 354391 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 354391 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 354398 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 354422 INFO  (zkCallback-414-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2ca4a4fa name:ZooKeeperConnection Watcher:127.0.0.1:39994/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 354422 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 354422 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 354427 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 354427 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C7E4E06A8290DA3E-001/control-001/solr.xml
   [junit4]   2> 354432 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C7E4E06A8290DA3E-001/control-001/cores
   [junit4]   2> 354433 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.CoreContainer New CoreContainer 943977700
   [junit4]   2> 354433 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C7E4E06A8290DA3E-001/control-001]
   [junit4]   2> 354433 WARN  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.CoreContainer Couldn't add files from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C7E4E06A8290DA3E-001/control-001/lib to classpath: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C7E4E06A8290DA3E-001/control-001/lib
   [junit4]   2> 354433 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,
   [junit4]   2> 354435 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.u.UpdateShardHandler Created UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&maxConnections=100000&maxConnectionsPerHost=100000
   [junit4]   2> 354435 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 354435 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 354435 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39994/solr
   [junit4]   2> 354436 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 354436 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 354438 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 354451 INFO  (zkCallback-417-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2890341e name:ZooKeeperConnection Watcher:127.0.0.1:39994 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 354454 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 354454 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 354455 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$65/1942881181@4c08a61c
   [junit4]   2> 354456 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 354465 INFO  (zkCallback-418-thread-1-processing-n:127.0.0.1:59258_h_ej%2Frc) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5609ff6a name:ZooKeeperConnection Watcher:127.0.0.1:39994/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 354465 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 354466 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
   [junit4]   2> 354468 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
   [junit4]   2> 354471 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
   [junit4]   2> 354473 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
   [junit4]   2> 354475 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
   [junit4]   2> 354476 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 354478 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 354484 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 354485 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 354486 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 354487 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.ZkStateReader Loaded cluster properties: {urlScheme=https}
   [junit4]   2> 354487 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 354489 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
   [junit4]   2> 354491 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
   [junit4]   2> 354492 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 354492 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/96524164962320391-127.0.0.1:59258_h_ej%2Frc-n_0000000000
   [junit4]   2> 354492 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:59258_h_ej%2Frc
   [junit4]   2> 354492 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
   [junit4]   2> 354493 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.Overseer Overseer (id=96524164962320391-127.0.0.1:59258_h_ej%2Frc-n_0000000000) starting
   [junit4]   2> 354495 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
   [junit4]   2> 354501 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=10000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 354501 INFO  (OverseerCollectionConfigSetProcessor-96524164962320391-127.0.0.1:59258_h_ej%2Frc-n_0000000000) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.O

[...truncated too long message...]

.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
   [junit4]    > 	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
   [junit4]    > 	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
   [junit4]    > 	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
   [junit4]    > 	... 10 more
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_C7E4E06A8290DA3E-001
   [junit4]   2> Sep 02, 2016 8:26:35 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {range_facet_l_dv=PostingsFormat(name=Memory doPackFST= true), _version_=FST50, multiDefault=PostingsFormat(name=LuceneFixedGap), intDefault=FST50, id=PostingsFormat(name=Memory doPackFST= true), range_facet_i_dv=FST50, range_facet_l=FST50, timestamp=FST50}, docValues:{range_facet_l_dv=DocValuesFormat(name=Asserting), range_facet_i_dv=DocValuesFormat(name=Memory), timestamp=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=801, maxMBSortInHeap=6.596519607923508, sim=RandomSimilarity(queryNorm=true): {}, locale=fr-CH, timezone=Africa/Windhoek
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=172398656,total=520093696
   [junit4]   2> NOTE: All tests run in this JVM: [TestBackupRepositoryFactory, AliasIntegrationTest, NumericFieldsTest, TestManagedSchemaAPI, SecurityConfHandlerTest, SpatialFilterTest, TestSolrCoreSnapshots, TestCollectionAPI, ExitableDirectoryReaderTest, SuggestComponentTest, ShardRoutingTest, TestTrie, DocValuesMissingTest, ZkStateWriterTest, SpellCheckComponentTest, HighlighterTest, AsyncMigrateRouteKeyTest, CoreAdminRequestStatusTest, WordBreakSolrSpellCheckerTest, UniqFieldsUpdateProcessorFactoryTest, CollectionTooManyReplicasTest, TestBM25SimilarityFactory, HdfsTlogReplayBufferedWhileIndexingTest, SchemaVersionSpecificBehaviorTest, SolrCmdDistributorTest, TestRandomFlRTGCloud, TestSchemaSimilarityResource, VersionInfoTest, TestDownShardTolerantSearch, TestDefaultSearchFieldResource, MultiTermTest, SortByFunctionTest, ZkCLITest, TestSolrCLIRunExample, SimpleMLTQParserTest, OpenExchangeRatesOrgProviderTest, TestSolrQueryParserDefaultOperatorResource, ConvertedLegacyTest, TestFieldCollectionResource, AnalysisErrorHandlingTest, TestFieldSortValues, LoggingHandlerTest, ClassificationUpdateProcessorFactoryTest, DOMUtilTest, TestLeaderElectionZkExpiry, DistributedTermsComponentTest, BJQParserTest, DistributedQueryComponentCustomSortTest, CreateCollectionCleanupTest, TestDFRSimilarityFactory, TestReqParamsAPI, BasicFunctionalityTest, DistanceUnitsTest, TestExactStatsCache, TestPivotHelperCode, SimplePostToolTest, DeleteLastCustomShardedReplicaTest, SolrPluginUtilsTest, OverriddenZkACLAndCredentialsProvidersTest, TestFieldTypeResource, FacetPivotSmallTest, TestQuerySenderListener, PeerSyncReplicationTest, TestDefaultStatsCache, TestScoreJoinQPScore, TestCollapseQParserPlugin, TestMacros, TestDistributedGrouping, TestConfigSetsAPIZkFailure, NotRequiredUniqueKeyTest, BooleanFieldTest, VMParamsZkACLAndCredentialsProvidersTest, SpellCheckCollatorTest, FileUtilsTest, TermsComponentTest, TestFieldCacheWithThreads, TestNRTOpen, TestFieldCache, DistribDocExpirationUpdateProcessorTest, HdfsNNFailoverTest, TestConfigReload, TestSolrConfigHandlerCloud, ShardSplitTest, MinimalSchemaTest, UpdateRequestProcessorFactoryTest, TestSolrConfigHandler, SuggesterFSTTest, TestWordDelimiterFilterFactory, ZkSolrClientTest, TestNonDefinedSimilarityFactory, TestFastLRUCache, TestHashPartitioner, DistributedMLTComponentTest, TestSubQueryTransformer, SolrInfoMBeanTest, TestConfigSetProperties, TestSystemIdResolver, ChaosMonkeySafeLeaderTest, HdfsRestartWhileUpdatingTest, DocumentBuilderTest, SyncSliceTest, OpenCloseCoreStressTest, ClusterStateUpdateTest, TestZkChroot, TestRandomDVFaceting, ShardRoutingCustomTest, AutoCommitTest, TestFaceting, TestRecovery, DistributedSpellCheckComponentTest, TermVectorComponentDistributedTest, ZkControllerTest, TestRealTimeGet, TestJoin, TestReload, TestStressVersions, TestCoreContainer, SimpleFacetsTest, StatsComponentTest, PeerSyncTest, TestSort, TestFiltering, TestFunctionQuery, TestLazyCores, DirectUpdateHandlerTest, SoftAutoCommitTest, ShowFileRequestHandlerTest, CurrencyFieldOpenExchangeTest, SolrIndexSplitterTest, TestCoreDiscovery, SolrRequestParserTest, TestFoldingMultitermQuery, DocValuesTest, SuggesterTSTTest, TestStressLucene, SuggesterWFSTTest, PolyFieldTest, NoCacheHeaderTest, TestUpdate, TestAtomicUpdateErrorCases, QueryEqualityTest, StatelessScriptUpdateProcessorFactoryTest, DistanceFunctionTest, TestSolrDeletionPolicy1, XsltUpdateRequestHandlerTest, DebugComponentTest, CacheHeaderTest, IndexBasedSpellCheckerTest, TestSurroundQueryParser, DisMaxRequestHandlerTest, TestQueryUtils, DirectSolrSpellCheckerTest, DocumentAnalysisRequestHandlerTest, TestQueryTypes, TermVectorComponentTest, TestIndexingPerformance, MoreLikeThisHandlerTest, TestArbitraryIndexDir, FastVectorHighlighterTest, RegexBoostProcessorTest, TestJmxIntegration, ReturnFieldsTest, JSONWriterTest, TestCSVResponseWriter, MBeansHandlerTest, QueryParsingTest, TestAnalyzedSuggestions, JsonLoaderTest, BinaryUpdateRequestHandlerTest, TestPartialUpdateDeduplication, TestPHPSerializedResponseWriter, CSVRequestHandlerTest, TestComponentsName, SearchHandlerTest, TestBinaryResponseWriter, TestLFUCache, PreAnalyzedFieldTest, TestSuggestSpellingConverter, SpellingQueryConverterTest, RAMDirectoryFactoryTest, TestLRUCache, TestUtils, TestDocumentBuilder, ZkNodePropsTest, SliceStateTest, UUIDFieldTest, TestRTGBase, DistributedIntervalFacetingTest, CdcrVersionReplicationTest, CleanupOldIndexTest, CloudExitableDirectoryReaderTest, CollectionReloadTest, ConcurrentDeleteAndCreateCollectionTest, DistribCursorPagingTest, DistribJoinFromCollectionTest, HttpPartitionTest, MigrateRouteKeyTest, OverseerModifyCollectionTest, OverseerRolesTest, ReplaceNodeTest, RollingRestartTest, TestStressLiveNodes, TestTolerantUpdateProcessorCloud, TestTolerantUpdateProcessorRandomCloud, TriLevelCompositeIdRoutingTest, HdfsChaosMonkeyNothingIsSafeTest, HdfsUnloadDistributedZkTest, StressHdfsTest, CoreSorterTest, HdfsDirectoryFactoryTest, TestConfigSets, TestImplicitCoreProperties, TestInfoStreamLogging, TestShardHandlerFactory, RequestLoggingTest, TestReplicationHandlerBackup, CoreAdminCreateDiscoverTest, DistributedExpandComponentTest, DistributedQueryComponentOptimizationTest, JavabinLoaderTest, HighlighterMaxOffsetTest, TestSlowCompositeReaderWrapper, CheckHdfsIndexTest, TestCloudSchemaless, TestReloadDeadlock, TestSearcherReuse, TestSmileRequest, TestStandardQParsers, TestStressUserVersions, TestXmlQParser, TestMinMaxOnMultiValuedField, TestOrdValues, TestSortByMinMaxFunction, BlockJoinFacetDistribTest, CloudMLTQParserTest, BasicAuthIntegrationTest]
   [junit4] Completed [596/632 (8!)] on J1 in 18.87s, 1 test, 1 error <<< FAILURES!

[...truncated 113 lines...]
   [junit4] JVM J2: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20160902_183833_957.sysout
   [junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: Java heap space
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid3526.hprof ...
   [junit4] Heap dump file created [491695500 bytes in 1.772 secs]
   [junit4] <<< JVM J2: EOF ----

[...truncated 3 lines...]
   [junit4] JVM J1: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20160902_183833_951.sysout
   [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid3521.hprof ...
   [junit4] Heap dump file created [560566947 bytes in 3.764 secs]
   [junit4] <<< JVM J1: EOF ----

[...truncated 9 lines...]
   [junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20160902_183833_950.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid3520.hprof ...
   [junit4] Heap dump file created [488680050 bytes in 3.901 secs]
   [junit4] <<< JVM J0: EOF ----

[...truncated 11036 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:715: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid3520.hprof
* java_pid3521.hprof
* java_pid3526.hprof

Total time: 318 minutes 23 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any