You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2016/08/27 17:45:54 UTC

[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1107 - Still Failing

Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1107/

10 tests failed.
FAILED:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
document count mismatch.  control=13246 sum(shards)=13247 cloudClient=13247

Stack Trace:
java.lang.AssertionError: document count mismatch.  control=13246 sum(shards)=13247 cloudClient=13247
	at __randomizedtesting.SeedInfo.seed([CE91A8BCC182042E:46C597666F7E69D6]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1323)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:231)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.TestLocalFSCloudBackupRestore.test

Error Message:
Error from server at http://127.0.0.1:34175/solr: specified location file:///location/does/not/exist does not exist.

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:34175/solr: specified location file:///location/does/not/exist does not exist.
	at __randomizedtesting.SeedInfo.seed([CE91A8BCC182042E:46C597666F7E69D6]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:608)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:415)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:367)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1280)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1050)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:992)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:166)
	at org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.testInvalidPath(AbstractCloudBackupRestoreTestCase.java:148)
	at org.apache.solr.cloud.AbstractCloudBackupRestoreTestCase.test(AbstractCloudBackupRestoreTestCase.java:127)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest

Error Message:
3 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest:     1) Thread[id=54954, name=searcherExecutor-11826-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=54900, name=searcherExecutor-11795-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=55101, name=searcherExecutor-11832-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 3 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest: 
   1) Thread[id=54954, name=searcherExecutor-11826-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=54900, name=searcherExecutor-11795-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=55101, name=searcherExecutor-11832-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([CE91A8BCC182042E]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=54954, name=searcherExecutor-11826-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=54900, name=searcherExecutor-11795-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=55101, name=searcherExecutor-11832-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=54954, name=searcherExecutor-11826-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=54900, name=searcherExecutor-11795-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=55101, name=searcherExecutor-11832-thread-1, state=WAITING, group=TGRP-HdfsCollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([CE91A8BCC182042E]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.handler.TestReplicationHandler

Error Message:
ObjectTracker found 2 object(s) that were not released!!! [NRTCachingDirectory, NRTCachingDirectory]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 2 object(s) that were not released!!! [NRTCachingDirectory, NRTCachingDirectory]
	at __randomizedtesting.SeedInfo.seed([CE91A8BCC182042E]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.GeneratedMethodAccessor19.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.util.TestSolrCLIRunExample

Error Message:
ObjectTracker found 4 object(s) that were not released!!! [SolrCore, MockDirectoryWrapper, RawDirectoryWrapper, RawDirectoryWrapper]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 4 object(s) that were not released!!! [SolrCore, MockDirectoryWrapper, RawDirectoryWrapper, RawDirectoryWrapper]
	at __randomizedtesting.SeedInfo.seed([CE91A8BCC182042E]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.GeneratedMethodAccessor47.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard1

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard1
	at __randomizedtesting.SeedInfo.seed([CE91A8BCC182042E:928C4F3D0FA2B018]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart(CdcrReplicationDistributedZkTest.java:264)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
	at __randomizedtesting.SeedInfo.seed([CE91A8BCC182042E]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.GeneratedMethodAccessor47.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest:     1) Thread[id=35887, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=35857, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=35877, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=35892, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=35847, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=35862, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=35852, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=35867, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=35872, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=35882, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 
   1) Thread[id=35887, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=35857, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=35877, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=35892, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=35847, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=35862, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=35852, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=35867, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=35872, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=35882, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([CE91A8BCC182042E]:0)


FAILED:  org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:35987/if_tk/v

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:35987/if_tk/v
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:438)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more




Build Log:
[...truncated 11829 lines...]
   [junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20160827_154205_790.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid6364.hprof ...
   [junit4] Heap dump file created [490073271 bytes in 2.385 secs]
   [junit4] <<< JVM J0: EOF ----

   [junit4] JVM J0: stderr was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20160827_154205_790.syserr
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] 
   [junit4] Exception: java.lang.OutOfMemoryError thrown from the UncaughtExceptionHandler in thread "Thread-34046"
   [junit4] WARN: Unhandled exception in event serialization. -> java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] <<< JVM J0: EOF ----

[...truncated 66 lines...]
   [junit4] Suite: org.apache.solr.handler.TestReplicationHandler
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/init-core-data-001
   [junit4]   2> 2020216 INFO  (SUITE-TestReplicationHandler-seed#[CE91A8BCC182042E]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
   [junit4]   2> 2020218 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.SolrTestCaseJ4 ###Starting doTestReplicateAfterStartup
   [junit4]   2> 2020218 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-001/collection1
   [junit4]   2> 2020222 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 2020223 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@20473399{/solr,null,AVAILABLE}
   [junit4]   2> 2020225 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.e.j.s.ServerConnector Started ServerConnector@199d6127{HTTP/1.1,[http/1.1]}{127.0.0.1:33253}
   [junit4]   2> 2020225 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.e.j.s.Server Started @2026603ms
   [junit4]   2> 2020225 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-001/collection1/data, hostContext=/solr, hostPort=33253}
   [junit4]   2> 2020226 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 2020226 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-001'
   [junit4]   2> 2020226 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2020226 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 2020226 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-001/solr.xml
   [junit4]   2> 2020231 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-001/.
   [junit4]   2> 2020231 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.CoreContainer New CoreContainer 800480747
   [junit4]   2> 2020231 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-001]
   [junit4]   2> 2020232 WARN  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.CoreContainer Couldn't add files from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-001/lib to classpath: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-001/lib
   [junit4]   2> 2020232 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,
   [junit4]   2> 2020234 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.u.UpdateShardHandler Created UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&maxConnections=100000&maxConnectionsPerHost=100000
   [junit4]   2> 2020234 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 2020234 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2020234 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 2020234 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 2020235 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-001/.
   [junit4]   2> 2020237 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection1, config=solrconfig.xml, loadOnStartup=true, schema=schema.xml, configSetProperties=configsetprops.json, transient=false, dataDir=data/}
   [junit4]   2> 2020237 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.CorePropertiesLocator Found core collection1 in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-001/./collection1
   [junit4]   2> 2020237 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions
   [junit4]   2> 2020243 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.s.SolrDispatchFilter user.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2
   [junit4]   2> 2020243 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 2020243 INFO  (coreLoadExecutor-5843-thread-1) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-001/collection1'
   [junit4]   2> 2020243 INFO  (coreLoadExecutor-5843-thread-1) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2020243 INFO  (coreLoadExecutor-5843-thread-1) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 2020249 INFO  (coreLoadExecutor-5843-thread-1) [    ] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 2020252 INFO  (coreLoadExecutor-5843-thread-1) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 2020265 INFO  (coreLoadExecutor-5843-thread-1) [    ] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2020268 INFO  (coreLoadExecutor-5843-thread-1) [    ] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 2020270 INFO  (coreLoadExecutor-5843-thread-1) [    ] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 2020272 INFO  (coreLoadExecutor-5843-thread-1) [    ] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properties: Can't find resource 'configsetprops.json' in classpath or '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-001/collection1'
   [junit4]   2> 2020272 INFO  (coreLoadExecutor-5843-thread-1) [    ] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-001/./collection1
   [junit4]   2> 2020272 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2020272 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-001/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-001/./collection1/data/]
   [junit4]   2> 2020272 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.c.SolrCore JMX monitoring not detected for core: collection1
   [junit4]   2> 2020273 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-001/./collection1/data/snapshot_metadata
   [junit4]   2> 2020273 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.c.s.SolrSnapshotMetaDataManager Loading from snapshot metadata file...
   [junit4]   2> 2020273 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-001/./collection1/data
   [junit4]   2> 2020273 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.c.SolrCore New index directory detected: old=null new=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-001/./collection1/data/index/
   [junit4]   2> 2020273 WARN  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1] Solr index directory '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-001/./collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 2020273 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-001/./collection1/data/index
   [junit4]   2> 2020274 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@34e99fe0
   [junit4]   2> 2020274 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@5f40f80f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1ea95ab9),segFN=segments_1,generation=1}
   [junit4]   2> 2020274 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 2020274 INFO  (OldIndexDirectoryCleanupThreadForCore-collection1) [    x:collection1] o.a.s.c.SolrCore Looking for old index directories to cleanup for core collection1 in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-001/./collection1/data/
   [junit4]   2> 2020276 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 2020277 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 2020278 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 2020279 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 2020279 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 2020281 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.c.PluginBag requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 2020281 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,standard,lazy,/update/json/docs,/admin/luke,/export,/get,/admin/properties,/update/json,/admin/threads,/update/csv,/sql,/graph,/admin/segments,/admin/system,/replication,defaults,/config,/stream,/schema,/admin/plugins,/admin/logging,/admin/ping,/update,/admin/file,/terms
   [junit4]   2> 2020281 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.c.SolrCore Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 2020281 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2020281 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2020281 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=48, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=2.84375, floorSegmentMB=1.8349609375, forceMergeDeletesPctAllowed=8.187649428171664, segmentsPerTier=48.0, maxCFSSegmentSizeMB=1.6044921875, noCFSRatio=1.0
   [junit4]   2> 2020281 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@5f40f80f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1ea95ab9),segFN=segments_1,generation=1}
   [junit4]   2> 2020281 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 2020282 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@231b4814[collection1] main]
   [junit4]   2> 2020282 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-001/collection1/conf
   [junit4]   2> 2020282 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.r.RestManager Initializing RestManager with initArgs: {storageDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-001/collection1/conf}
   [junit4]   2> 2020282 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using file:dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-001/collection1/conf
   [junit4]   2> 2020282 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 2020282 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Replication enabled for following config files: schema.xml,xslt/dummy.xsl
   [junit4]   2> 2020282 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 2020283 INFO  (searcherExecutor-5844-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@231b4814[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2020283 INFO  (coreLoadExecutor-5843-thread-1) [    x:collection1] o.a.s.c.CoreContainer registering core: collection1
   [junit4]   2> 2020752 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/collection1
   [junit4]   2> 2020754 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 2020761 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7b66e8b0{/solr,null,AVAILABLE}
   [junit4]   2> 2020761 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.e.j.s.ServerConnector Started ServerConnector@1346ce25{HTTP/1.1,[http/1.1]}{127.0.0.1:46622}
   [junit4]   2> 2020761 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.e.j.s.Server Started @2027139ms
   [junit4]   2> 2020761 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/collection1/data, hostContext=/solr, hostPort=46622}
   [junit4]   2> 2020762 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 2020762 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002'
   [junit4]   2> 2020762 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2020762 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 2020762 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/solr.xml
   [junit4]   2> 2020766 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/.
   [junit4]   2> 2020766 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.CoreContainer New CoreContainer 1375012440
   [junit4]   2> 2020766 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002]
   [junit4]   2> 2020766 WARN  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.CoreContainer Couldn't add files from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/lib to classpath: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/lib
   [junit4]   2> 2020767 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,
   [junit4]   2> 2020768 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.u.UpdateShardHandler Created UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&maxConnections=100000&maxConnectionsPerHost=100000
   [junit4]   2> 2020768 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 2020769 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2020769 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 2020769 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 2020769 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/.
   [junit4]   2> 2020770 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection1, config=solrconfig.xml, loadOnStartup=true, schema=schema.xml, configSetProperties=configsetprops.json, transient=false, dataDir=data/}
   [junit4]   2> 2020770 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.CorePropertiesLocator Found core collection1 in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/./collection1
   [junit4]   2> 2020770 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions
   [junit4]   2> 2020771 INFO  (coreLoadExecutor-5853-thread-1) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/collection1'
   [junit4]   2> 2020771 INFO  (coreLoadExecutor-5853-thread-1) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2020771 INFO  (coreLoadExecutor-5853-thread-1) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 2020775 INFO  (coreLoadExecutor-5853-thread-1) [    ] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 2020778 INFO  (coreLoadExecutor-5853-thread-1) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 2020778 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.s.SolrDispatchFilter user.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2
   [junit4]   2> 2020778 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 2020783 INFO  (coreLoadExecutor-5853-thread-1) [    ] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2020785 INFO  (coreLoadExecutor-5853-thread-1) [    ] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 2020787 INFO  (coreLoadExecutor-5853-thread-1) [    ] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 2020788 INFO  (coreLoadExecutor-5853-thread-1) [    ] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properties: Can't find resource 'configsetprops.json' in classpath or '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/collection1'
   [junit4]   2> 2020788 INFO  (coreLoadExecutor-5853-thread-1) [    ] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/./collection1
   [junit4]   2> 2020788 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2020789 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/./collection1/data/]
   [junit4]   2> 2020789 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.c.SolrCore JMX monitoring not detected for core: collection1
   [junit4]   2> 2020789 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/./collection1/data/snapshot_metadata
   [junit4]   2> 2020789 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.c.s.SolrSnapshotMetaDataManager Loading from snapshot metadata file...
   [junit4]   2> 2020789 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/./collection1/data
   [junit4]   2> 2020789 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.c.SolrCore New index directory detected: old=null new=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/./collection1/data/index/
   [junit4]   2> 2020789 WARN  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1] Solr index directory '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/./collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 2020790 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/./collection1/data/index
   [junit4]   2> 2020790 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@32a4126c
   [junit4]   2> 2020790 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@25eb1cb3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@11f8dffe),segFN=segments_1,generation=1}
   [junit4]   2> 2020790 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 2020791 INFO  (OldIndexDirectoryCleanupThreadForCore-collection1) [    x:collection1] o.a.s.c.SolrCore Looking for old index directories to cleanup for core collection1 in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/./collection1/data/
   [junit4]   2> 2020792 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 2020793 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 2020794 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 2020794 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 2020795 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 2020796 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.c.PluginBag requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 2020796 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,standard,lazy,/update/json/docs,/admin/luke,/export,/get,/admin/properties,/update/json,/admin/threads,/update/csv,/sql,/graph,/admin/segments,/admin/system,/replication,defaults,/config,/stream,/schema,/admin/plugins,/admin/logging,/admin/ping,/update,/admin/file,/terms
   [junit4]   2> 2020796 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.c.SolrCore Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 2020796 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 2020796 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 2020797 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=48, maxMergeAtOnceExplicit=48, maxMergedSegmentMB=2.84375, floorSegmentMB=1.8349609375, forceMergeDeletesPctAllowed=8.187649428171664, segmentsPerTier=48.0, maxCFSSegmentSizeMB=1.6044921875, noCFSRatio=1.0
   [junit4]   2> 2020797 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@25eb1cb3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@11f8dffe),segFN=segments_1,generation=1}
   [junit4]   2> 2020797 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 2020797 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2875c7a[collection1] main]
   [junit4]   2> 2020797 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/collection1/conf
   [junit4]   2> 2020797 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.r.RestManager Initializing RestManager with initArgs: {storageDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/collection1/conf}
   [junit4]   2> 2020797 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using file:dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/collection1/conf
   [junit4]   2> 2020798 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 2020807 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Poll scheduled at an interval of 1000ms
   [junit4]   2> 2020807 INFO  (coreLoadExecutor-5853-thread-1) [    x:collection1] o.a.s.c.CoreContainer registering core: collection1
   [junit4]   2> 2020807 INFO  (searcherExecutor-5854-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@2875c7a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2020812 INFO  (qtp329750535-35946) [    x:collection1] o.a.s.c.S.Request [collection1]  webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0
   [junit4]   2> 2021280 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.e.j.s.ServerConnector Stopped ServerConnector@1346ce25{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 2021280 INFO  (TEST-TestReplicationHandler.doTestReplicateAfterStartup-seed#[CE91A8BCC182042E]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1375012440
   [junit4]   2> 2021283 INFO  (coreCloseExecutor-5859-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@243f93e
   [junit4]   2> 2021283 INFO  (coreCloseExecutor-5859-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 2021283 INFO  (coreCloseExecutor-5859-thread-1) [    x:collection1] o.a.s.u.SolrCoreState Closing SolrCoreState
   [junit4]   2> 2021283 INFO  (coreCloseExecutor-5859-thread-1) [    x:collection1] o.a.s.u.DefaultSolrCoreState SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 2021283 INFO  (coreCloseExecutor-5859-thread-1) [    x:collection1] o.a.s.u.DefaultSolrCoreState closing IndexWriter with IndexWriterCloser
   [junit4]   2> 2021284 INFO  (coreCloseExecutor-5859-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1] Closing main searcher on request.
   [junit4]   2> 2021284 INFO  (coreCloseExecutor-5859-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory Closing MockDirectoryFactory - 3 directories currently being tracked
   [junit4]   2> 2021284 INFO  (coreCloseExecutor-5859-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory looking to close /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/./collection1/data/index [CachedDir<<refCount=0;path=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/./collection1/data/index;done=false>>]
   [junit4]   2> 2021284 INFO  (coreCloseExecutor-5859-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory Closing directory, CoreContainer#isShutdown=true
   [junit4]   2> 2021284 INFO  (coreCloseExecutor-5859-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory Closing directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/./collection1/data/index
   [junit4]   2> 2021285 INFO  (coreCloseExecutor-5859-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory looking to close /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/./collection1/data/snapshot_metadata [CachedDir<<refCount=0;path=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/./collection1/data/snapshot_metadata;done=false>>]
   [junit4]   2> 2021285 INFO  (coreCloseExecutor-5859-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory Closing directory, CoreContainer#isShutdown=true
   [junit4]   2> 2021285 INFO  (coreCloseExecutor-5859-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory Closing directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/./collection1/data/snapshot_metadata
   [junit4]   2> 2021285 INFO  (coreCloseExecutor-5859-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory looking to close /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler_CE91A8BCC182042E-001/solr-instance-002/./collection1/data [CachedDir<<refCount=0;path=/x1

[...truncated too long message...]

b/apacheds-interceptors-event-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/bcprov-jdk15-1.45.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/cglib-nodep-2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/commons-math3-3.4.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/easymock-3.0.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-common-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-hdfs-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-minikdc-2.7.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-annotations-2.5.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-core-asl-1.9.13.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-databind-2.5.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-mapper-asl-1.9.13.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/mockito-core-1.9.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/objenesis-1.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/presto-parser-0.122.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/slice-0.10.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/analysis/icu/lucene-analyzers-icu-7.0.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/analysis-extras/lib/icu4j-56.1.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_102/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.3.4.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20160827_154205_790.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20160827_154205_790.suites -stdin
   [junit4] ERROR: JVM J0 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1501)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:121)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:955)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:952)
   [junit4] 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4] 	at java.lang.Thread.run(Thread.java:745)

BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:707: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build.xml:233: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/common-build.xml:536: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:1443: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:984: At least one slave process threw an exception, first: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.

Total time: 329 minutes 48 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1128 - Failure

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1128/

8 tests failed.
FAILED:  org.apache.lucene.index.TestDemoParallelLeafReader.testRandomMultipleSchemaGensSameField

Error Message:
cannot change field "number" from points dimensionCount=1, numBytes=8 to inconsistent dimensionCount=0, numBytes=0

Stack Trace:
java.lang.IllegalArgumentException: cannot change field "number" from points dimensionCount=1, numBytes=8 to inconsistent dimensionCount=0, numBytes=0
	at __randomizedtesting.SeedInfo.seed([7856CAB64594B856:2807C177B262E94F]:0)
	at org.apache.lucene.index.FieldInfo.update(FieldInfo.java:149)
	at org.apache.lucene.index.FieldInfos$Builder.addOrUpdateInternal(FieldInfos.java:412)
	at org.apache.lucene.index.FieldInfos$Builder.add(FieldInfos.java:432)
	at org.apache.lucene.index.SegmentMerger.mergeFieldInfos(SegmentMerger.java:186)
	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:84)
	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4287)
	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3864)
	at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
	at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
	at org.apache.lucene.index.IndexWriter.prepareCommitInternal(IndexWriter.java:3047)
	at org.apache.lucene.index.IndexWriter.commitInternal(IndexWriter.java:3181)
	at org.apache.lucene.index.IndexWriter.commit(IndexWriter.java:3146)
	at org.apache.lucene.index.TestDemoParallelLeafReader$ReindexingReader.commit(TestDemoParallelLeafReader.java:160)
	at org.apache.lucene.index.TestDemoParallelLeafReader.testRandomMultipleSchemaGensSameField(TestDemoParallelLeafReader.java:1091)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationStartStop

Error Message:
Timeout while trying to assert number of documents @ target_collection

Stack Trace:
java.lang.AssertionError: Timeout while trying to assert number of documents @ target_collection
	at __randomizedtesting.SeedInfo.seed([96E63FAC05C47E13:1525E13F111CF59A]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:273)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationStartStop(CdcrReplicationDistributedZkTest.java:173)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.AssertionError: expected:<10> but was:<4>
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:264)
	... 42 more


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testResilienceWithDeleteByQueryOnTarget

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard1

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard1
	at __randomizedtesting.SeedInfo.seed([96E63FAC05C47E13:3684AD36D59DBB56]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testResilienceWithDeleteByQueryOnTarget(CdcrReplicationDistributedZkTest.java:588)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.core.HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig

Error Message:
The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~134.217728 MB.

Stack Trace:
java.lang.RuntimeException: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~134.217728 MB.
	at __randomizedtesting.SeedInfo.seed([96E63FAC05C47E13:6149F687F84D9438]:0)
	at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:304)
	at org.apache.solr.core.HdfsDirectoryFactory.getBlockDirectoryCache(HdfsDirectoryFactory.java:280)
	at org.apache.solr.core.HdfsDirectoryFactory.create(HdfsDirectoryFactory.java:220)
	at org.apache.solr.core.HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig(HdfsDirectoryFactoryTest.java:108)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.OutOfMemoryError: Direct buffer memory
	at java.nio.Bits.reserveMemory(Bits.java:693)
	at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123)
	at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)
	at org.apache.solr.store.blockcache.BlockCache.<init>(BlockCache.java:63)
	at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:302)
	... 42 more


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestStressReplication

Error Message:
timed out waiting for collection1 startAt time to exceed: Thu Oct 13 11:49:29 OMST 2016

Stack Trace:
java.lang.AssertionError: timed out waiting for collection1 startAt time to exceed: Thu Oct 13 11:49:29 OMST 2016
	at __randomizedtesting.SeedInfo.seed([96E63FAC05C47E13:4D4D3F6A00EC17A0]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.handler.TestReplicationHandler.watchCoreStartAt(TestReplicationHandler.java:1508)
	at org.apache.solr.handler.TestReplicationHandler.doTestStressReplication(TestReplicationHandler.java:858)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:830)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:830)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.

[...truncated too long message...]

.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-collective-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-event-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/bcprov-jdk15-1.45.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/cglib-nodep-2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/commons-math3-3.4.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/easymock-3.0.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-common-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-hdfs-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-minikdc-2.7.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-annotations-2.5.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-databind-2.5.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/mockito-core-1.9.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/objenesis-1.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/presto-parser-0.122.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/slice-0.10.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/analysis/icu/lucene-analyzers-icu-7.0.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/analysis-extras/lib/icu4j-56.1.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_102/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.3.4.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20161013_051024_079.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20161013_051024_079.suites -stdin
   [junit4] ERROR: JVM J1 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1501)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:121)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:955)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:952)
   [junit4] 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4] 	at java.lang.Thread.run(Thread.java:745)

BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:773: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:709: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build.xml:252: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/common-build.xml:536: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:1443: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:984: At least one slave process threw an exception, first: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.

Total time: 255 minutes 38 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1127 - Still unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1127/

8 tests failed.
FAILED:  org.apache.lucene.search.TestFuzzyQuery.testRandom

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([DC469116CE6EC137]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.search.TestFuzzyQuery

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([DC469116CE6EC137]:0)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchAddsWithDelete

Error Message:
Timeout while trying to assert number of documents @ target_collection

Stack Trace:
java.lang.AssertionError: Timeout while trying to assert number of documents @ target_collection
	at __randomizedtesting.SeedInfo.seed([1CF269DB07F9BC4A:6BE8107FAFE0BD66]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:273)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchAddsWithDelete(CdcrReplicationDistributedZkTest.java:532)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.AssertionError: expected:<59> but was:<33>
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:264)
	... 42 more


FAILED:  org.apache.solr.cloud.hdfs.StressHdfsTest.test

Error Message:
Could not find collection:delete_data_dir

Stack Trace:
java.lang.AssertionError: Could not find collection:delete_data_dir
	at __randomizedtesting.SeedInfo.seed([1CF269DB07F9BC4A:94A65601A905D1B2]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:153)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:138)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:133)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:856)
	at org.apache.solr.cloud.hdfs.StressHdfsTest.createAndDeleteCollection(StressHdfsTest.java:159)
	at org.apache.solr.cloud.hdfs.StressHdfsTest.test(StressHdfsTest.java:103)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:830)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:830)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:830)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:830)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:830)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:267)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:214)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:183)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:167)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carr

[...truncated too long message...]

nterceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:430)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
   [junit4]   2> 	at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
   [junit4]   2> 	at org.apache.hadoop.metrics2.util.MBeans.unregister(MBeans.java:81)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stopMBeans(MetricsSourceAdapter.java:226)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stop(MetricsSourceAdapter.java:211)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stopSources(MetricsSystemImpl.java:463)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
   [junit4]   2> 	at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
   [junit4]   2> 	at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
   [junit4]   2> 	at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:198)
   [junit4]   2> 	at org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.teardownClass(HdfsCollectionsAPIDistributedZkTest.java:52)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: java.lang.NullPointerException
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
   [junit4]   2> 	... 54 more
   [junit4]   2> 4893991 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[1CF269DB07F9BC4A]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_1CF269DB07F9BC4A-001
   [junit4]   2> Oct 12, 2016 8:49:32 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 35 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=6185, maxDocsPerChunk=8, blockSize=992), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=6185, blockSize=992)), sim=RandomSimilarity(queryNorm=false): {}, locale=es-NI, timezone=Pacific/Wake
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=2,free=153867744,total=482869248
   [junit4]   2> NOTE: All tests run in this JVM: [HdfsNNFailoverTest, BlockDirectoryTest, TestDistribIDF, DistributedFacetPivotLongTailTest, TestQuerySenderNoQuery, MultiThreadedOCPTest, TestSolrConfigHandlerConcurrent, PrimUtilsTest, WrapperMergePolicyFactoryTest, HdfsChaosMonkeyNothingIsSafeTest, TestLRUStatsCache, TestPhraseSuggestions, SolrPluginUtilsTest, TestLeaderElectionZkExpiry, HdfsTlogReplayBufferedWhileIndexingTest, XmlUpdateRequestHandlerTest, TestReplicaProperties, GraphQueryTest, ParsingFieldUpdateProcessorsTest, TestGraphMLResponseWriter, DistributedExpandComponentTest, TestCloudSchemaless, HLLSerializationTest, EchoParamsTest, CollectionStateFormat2Test, TestBulkSchemaConcurrent, RAMDirectoryFactoryTest, DistribCursorPagingTest, TestReversedWildcardFilterFactory, TestSolrJ, CoreMergeIndexesAdminHandlerTest, TolerantUpdateProcessorTest, TestTolerantSearch, PreAnalyzedFieldTest, TestRandomCollapseQParserPlugin, SuggesterWFSTTest, SliceStateTest, TestReplicationHandlerBackup, CdcrUpdateLogTest, TestSchemaSimilarityResource, FullSolrCloudDistribCmdsTest, TestConfigSetsAPI, TestJsonFacetRefinement, DirectUpdateHandlerOptimizeTest, TestTolerantUpdateProcessorRandomCloud, TestGraphTermsQParserPlugin, TestSolrDeletionPolicy1, TestExtendedDismaxParser, TestIBSimilarityFactory, AssignTest, TestIndexingPerformance, TestAuthenticationFramework, DistributedFacetPivotWhiteBoxTest, TestManagedStopFilterFactory, ConnectionManagerTest, SegmentsInfoRequestHandlerTest, TestSolrXml, JavabinLoaderTest, BooleanFieldTest, HdfsLockFactoryTest, DocValuesMissingTest, TestWriterPerf, DistributedQueueTest, TestSolrFieldCacheMBean, CdcrBootstrapTest, TestCSVResponseWriter, TestSolrCloudWithDelegationTokens, TestFreeTextSuggestions, TestUtils, TestRandomFlRTGCloud, ShowFileRequestHandlerTest, LeaderInitiatedRecoveryOnCommitTest, TestComponentsName, TestAuthorizationFramework, TestSolrQueryParser, ConnectionReuseTest, HighlighterConfigTest, TestFieldTypeResource, StatsComponentTest, SuggesterTSTTest, RecoveryZkTest, TestGroupingSearch, TestManagedResource, SimpleFacetsTest, TestInitParams, OverseerTest, AddSchemaFieldsUpdateProcessorFactoryTest, PrimitiveFieldTypeTest, TestConfigSetProperties, TestLMDirichletSimilarityFactory, MBeansHandlerTest, TestSimpleQParserPlugin, TestExpandComponent, CleanupOldIndexTest, CurrencyFieldXmlFileTest, OverseerCollectionConfigSetProcessorTest, TestLegacyFieldCache, AnalyticsQueryTest, TestLegacyNumericRangeQueryBuilder, TestDefaultSearchFieldResource, TestBulkSchemaAPI, TestFastOutputStream, UpdateParamsTest, BasicDistributedZkTest, BasicDistributedZk2Test, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest, TestCoreDiscovery, CoreAdminHandlerTest, SuggesterTest, SpatialFilterTest, PolyFieldTest, WordBreakSolrSpellCheckerTest, FieldMutatingUpdateProcessorTest, DefaultValueUpdateProcessorTest, DocValuesMultiTest, DistanceFunctionTest, SolrInfoMBeanTest, DebugComponentTest, LukeRequestHandlerTest, StandardRequestHandlerTest, FileBasedSpellCheckerTest, TestValueSourceCache, PathHierarchyTokenizerFactoryTest, MoreLikeThisHandlerTest, TestJmxIntegration, ReturnFieldsTest, TestAnalyzedSuggestions, TestPartialUpdateDeduplication, PingRequestHandlerTest, TestLFUCache, SolrTestCaseJ4Test, TestCrossCoreJoin, TestCursorMarkWithoutUniqueKey, TestDistributedMissingSort, CdcrRequestHandlerTest, CloudExitableDirectoryReaderTest, CollectionReloadTest, CollectionsAPISolrJTest, DeleteReplicaTest, LeaderInitiatedRecoveryOnShardRestartTest, OverseerRolesTest, RollingRestartTest, TestDistribDocBasedVersion, TestExclusionRuleCollectionAccess, TestLeaderRecoverFromLogOnStartup, TestSSLRandomization, TestSizeLimitedDistributedMap, TestSolrCloudWithSecureImpersonation, TestStressLiveNodes, VMParamsZkACLAndCredentialsProvidersTest, HdfsBasicDistributedZk2Test, HdfsChaosMonkeySafeLeaderTest, HdfsCollectionsAPIDistributedZkTest]
   [junit4] Completed [571/638 (5!)] on J2 in 949.50s, 1 test, 1 error <<< FAILURES!

[...truncated 60518 lines...]



[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1126 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1126/

No tests ran.

Build Log:
[...truncated 11625 lines...]
   [junit4] JVM J1: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20161005_083740_205.sysout
   [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid6760.hprof ...
   [junit4] Heap dump file created [714278651 bytes in 16.125 secs]
   [junit4] <<< JVM J1: EOF ----

   [junit4] JVM J1: stderr was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20161005_083740_205.syserr
   [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
   [junit4] WARN: Unhandled exception in event serialization. -> java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] <<< JVM J1: EOF ----

[...truncated 32 lines...]
   [junit4] Suite: org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/init-core-data-001
   [junit4]   2> 1786063 INFO  (SUITE-ConcurrentDeleteAndCreateCollectionTest-seed#[BF6C5C74A9CEFC3D]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 1786071 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[BF6C5C74A9CEFC3D]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testConcurrentCreateAndDeleteOverTheSameConfig
   [junit4]   2> 1786071 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[BF6C5C74A9CEFC3D]) [    ] o.a.s.c.MiniSolrCloudCluster Starting cluster of 1 servers in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001
   [junit4]   2> 1786071 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[BF6C5C74A9CEFC3D]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 1786076 INFO  (Thread-1725) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1786076 INFO  (Thread-1725) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 1786179 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[BF6C5C74A9CEFC3D]) [    ] o.a.s.c.ZkTestServer start zk server on port:58815
   [junit4]   2> 1786239 INFO  (jetty-launcher-799-thread-1) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 1786283 INFO  (jetty-launcher-799-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@d571903{/solr,null,AVAILABLE}
   [junit4]   2> 1786286 INFO  (jetty-launcher-799-thread-1) [    ] o.e.j.s.ServerConnector Started ServerConnector@3d714097{SSL,[ssl, http/1.1]}{127.0.0.1:56041}
   [junit4]   2> 1786286 INFO  (jetty-launcher-799-thread-1) [    ] o.e.j.s.Server Started @1797249ms
   [junit4]   2> 1786286 INFO  (jetty-launcher-799-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=56041}
   [junit4]   2> 1786286 INFO  (jetty-launcher-799-thread-1) [    ] o.a.s.s.SolrDispatchFilter  ___      _       Welcome to Apache Solr? version 7.0.0
   [junit4]   2> 1786287 INFO  (jetty-launcher-799-thread-1) [    ] o.a.s.s.SolrDispatchFilter / __| ___| |_ _   Starting in cloud mode on port null
   [junit4]   2> 1786287 INFO  (jetty-launcher-799-thread-1) [    ] o.a.s.s.SolrDispatchFilter \__ \/ _ \ | '_|  Install dir: null
   [junit4]   2> 1786287 INFO  (jetty-launcher-799-thread-1) [    ] o.a.s.s.SolrDispatchFilter |___/\___/_|_|    Start time: 2016-10-05T09:07:37.802Z
   [junit4]   2> 1786344 INFO  (jetty-launcher-799-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 1786404 INFO  (jetty-launcher-799-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:58815/solr
   [junit4]   2> 1786528 INFO  (jetty-launcher-799-thread-1) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:56041_solr
   [junit4]   2> 1786529 INFO  (jetty-launcher-799-thread-1) [n:127.0.0.1:56041_solr    ] o.a.s.c.Overseer Overseer (id=96708752676749315-127.0.0.1:56041_solr-n_0000000000) starting
   [junit4]   2> 1786580 INFO  (jetty-launcher-799-thread-1) [n:127.0.0.1:56041_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:56041_solr
   [junit4]   2> 1786591 INFO  (zkCallback-805-thread-1-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 1786607 INFO  (jetty-launcher-799-thread-1) [n:127.0.0.1:56041_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/.
   [junit4]   2> 1786937 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=testconfig&name=collection0&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1786948 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=testconfig&name=collection1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1786983 INFO  (OverseerThreadFactory-2254-thread-1-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection0
   [junit4]   2> 1786992 INFO  (OverseerThreadFactory-2254-thread-2-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 1787172 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=testconfig&name=collection0_shard1_replica1&action=CREATE&numShards=1&collection=collection0&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1787254 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=testconfig&name=collection1_shard1_replica1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1787280 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1787395 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1788222 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1788239 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.s.IndexSchema [collection0_shard1_replica1] Schema name=minimal
   [junit4]   2> 1788246 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1788247 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'collection0_shard1_replica1' using configuration from collection collection0
   [junit4]   2> 1788252 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore [[collection0_shard1_replica1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/collection0_shard1_replica1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/./collection0_shard1_replica1/data/]
   [junit4]   2> 1788282 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1788304 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.IndexSchema [collection1_shard1_replica1] Schema name=minimal
   [junit4]   2> 1788306 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1788317 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica1' using configuration from collection collection1
   [junit4]   2> 1788318 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [[collection1_shard1_replica1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/collection1_shard1_replica1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/./collection1_shard1_replica1/data/]
   [junit4]   2> 1788335 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1788336 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1788337 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@391d9575[collection0_shard1_replica1] main]
   [junit4]   2> 1788355 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/testconfig
   [junit4]   2> 1788359 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/testconfig
   [junit4]   2> 1788360 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1788360 INFO  (searcherExecutor-2257-thread-1-processing-n:127.0.0.1:56041_solr x:collection0_shard1_replica1 s:shard1 c:collection0 r:core_node1) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore [collection0_shard1_replica1] Registered new searcher Searcher@391d9575[collection0_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1788388 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1788388 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1788390 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@7fce5056[collection1_shard1_replica1] main]
   [junit4]   2> 1788395 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1788395 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1788395 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:56041/solr/collection0_shard1_replica1/
   [junit4]   2> 1788396 ERROR (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SyncStrategy No UpdateLog found - cannot sync
   [junit4]   2> 1788402 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 1788406 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/testconfig
   [junit4]   2> 1788408 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/testconfig
   [junit4]   2> 1788408 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1788408 INFO  (searcherExecutor-2258-thread-1-processing-n:127.0.0.1:56041_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [collection1_shard1_replica1] Registered new searcher Searcher@7fce5056[collection1_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1788416 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:56041/solr/collection0_shard1_replica1/ shard1
   [junit4]   2> 1788420 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1788420 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1788420 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:56041/solr/collection1_shard1_replica1/
   [junit4]   2> 1788420 ERROR (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SyncStrategy No UpdateLog found - cannot sync
   [junit4]   2> 1788421 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 1788429 INFO  (zkCallback-805-thread-2-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1788433 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:56041/solr/collection1_shard1_replica1/ shard1
   [junit4]   2> 1788471 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1788485 INFO  (zkCallback-805-thread-2-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1788486 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=testconfig&name=collection0_shard1_replica1&action=CREATE&numShards=1&collection=collection0&shard=shard1&wt=javabin&version=2} status=0 QTime=1314
   [junit4]   2> 1788503 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1788535 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1788543 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=testconfig&name=collection1_shard1_replica1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2} status=0 QTime=1289
   [junit4]   2> 1788546 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1788547 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1788685 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1788991 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000000 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1788991 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000002 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1789503 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=testconfig&name=collection0&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2566
   [junit4]   2> 1789506 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=collection0&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1789510 INFO  (OverseerThreadFactory-2254-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 1789511 INFO  (qtp1908042262-5757) [n:127.0.0.1:56041_solr    ] o.a.s.c.SolrCore [collection0_shard1_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@33e50332
   [junit4]   2> 1789535 INFO  (qtp1908042262-5757) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=collection0_shard1_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=24
   [junit4]   2> 1789547 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=testconfig&name=collection1&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2599
   [junit4]   2> 1789552 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=collection1&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1789571 INFO  (OverseerThreadFactory-2254-thread-4-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 1789580 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr    ] o.a.s.c.SolrCore [collection1_shard1_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@79482cd9
   [junit4]   2> 1789594 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=collection1_shard1_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=14
   [junit4]   2> 1790273 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=collection0&action=DELETE&wt=javabin&version=2} status=0 QTime=767
   [junit4]   2> 1790274 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=testconfig&name=collection0&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1790288 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000004 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1790288 INFO  (OverseerThreadFactory-2254-thread-5-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection0
   [junit4]   2> 1790449 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=collection1&action=DELETE&wt=javabin&version=2} status=0 QTime=897
   [junit4]   2> 1790452 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=testconfig&name=collection1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1790484 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000006 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1790486 INFO  (OverseerThreadFactory-2254-thread-4-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 1790500 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=testconfig&name=collection0_shard1_replica1&action=CREATE&numShards=1&collection=collection0&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1790590 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=testconfig&name=collection1_shard1_replica1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1790592 INFO  (zkCallback-805-thread-1-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1790696 INFO  (zkCallback-805-thread-1-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1791543 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1791558 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.s.IndexSchema [collection0_shard1_replica1] Schema name=minimal
   [junit4]   2> 1791560 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1791562 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'collection0_shard1_replica1' using configuration from collection collection0
   [junit4]   2> 1791562 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore [[collection0_shard1_replica1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/collection0_shard1_replica1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/./collection0_shard1_replica1/data/]
   [junit4]   2> 1791575 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1791576 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1791580 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@aac94c4[collection0_shard1_replica1] main]
   [junit4]   2> 1791581 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/testconfig
   [junit4]   2> 1791582 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/testconfig
   [junit4]   2> 1791582 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1791582 INFO  (searcherExecutor-2265-thread-1-processing-n:127.0.0.1:56041_solr x:collection0_shard1_replica1 s:shard1 c:collection0 r:core_node1) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore [collection0_shard1_replica1] Registered new searcher Searcher@aac94c4[collection0_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1791587 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1791588 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1791588 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:56041/solr/collection0_shard1_replica1/
   [junit4]   2> 1791588 ERROR (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SyncStrategy No UpdateLog found - cannot sync
   [junit4]   2> 1791588 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 1791600 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1791601 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:56041/solr/collection0_shard1_replica1/ shard1
   [junit4]   2> 1791608 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.IndexSchema [collection1_shard1_replica1] Schema name=minimal
   [junit4]   2> 1791610 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1791611 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica1' using configuration from collection collection1
   [junit4]   2> 1791612 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [[collection1_shard1_replica1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/collection1_shard1_replica1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/./collection1_shard1_replica1/data/]
   [junit4]   2> 1791623 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1791623 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1791624 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@35b94579[collection1_shard1_replica1] main]
   [junit4]   2> 1791625 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/testconfig
   [junit4]   2> 1791625 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/testconfig
   [junit4]   2> 1791625 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1791626 INFO  (searcherExecutor-2269-thread-1-processing-n:127.0.0.1:56041_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [collection1_shard1_replica1] Registered new searcher Searcher@35b94579[collection1_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1791631 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1791632 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1791632 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1791632 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:56041/solr/collection1_shard1_replica1/
   [junit4]   2> 1791632 ERROR (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SyncStrategy No UpdateLog found - cannot sync
   [junit4]   2> 1791632 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 1791638 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:56041/solr/collection1_shard1_replica1/ shard1
   [junit4]   2> 1791652 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1791655 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=testconfig&name=collection0_shard1_replica1&action=CREATE&numShards=1&collection=collection0&shard=shard1&wt=javabin&version=2} status=0 QTime=1155
   [junit4]   2> 1791657 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1791668 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1791691 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1791700 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1791703 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=testconfig&name=collection1_shard1_replica1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2} status=0 QTime=1113
   [junit4]   2> 1791709 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1791804 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1792488 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000008 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1792488 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000010 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1792668 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=testconfig&name=collection0&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2394
   [junit4]   2> 1792669 INFO  (qtp1908042262-5757) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=collection0&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1792680 INFO  (OverseerThreadFactory-2254-thread-4-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 1792682 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr    ] o.a.s.c.SolrCore [collection0_shard1_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@4e26518e
   [junit4]   2> 1792689 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=collection0_shard1_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=8
   [junit4]   2> 1792711 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=testconfig&name=collection1&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2259
   [junit4]   2> 1792712 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=collection1&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1792736 INFO  (OverseerThreadFactory-2254-thread-5-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 1792737 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr    ] o.a.s.c.SolrCore [collection1_shard1_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@f4d8dc4
   [junit4]   2> 1792740 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=collection1_shard1_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=2
   [junit4]   2> 1793301 INFO  (qtp1908042262-5757) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=collection0&action=DELETE&wt=javabin&version=2} status=0 QTime=631
   [junit4]   2> 1793303 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=testconfig&name=collection0&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1793340 INFO  (OverseerThreadFactory-2254-thread-4-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection0
   [junit4]   2> 1793348 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000012 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1793453 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=collection1&action=DELETE&wt=javabin&version=2} status=0 QTime=740
   [junit4]   2> 1793454 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=testconfig&name=collection1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1793463 INFO  (OverseerThreadFactory-2254-thread-5-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 1793464 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000014 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1793572 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=testconfig&name=collection0_shard1_replica1&action=CREATE&numShards=1&collection=collection0&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1793676 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=testconfig&name=collection1_shard1_replica1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1793679 INFO  (zkCallback-805-thread-1-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1793783 INFO  (zkCallback-805-thread-1-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1794585 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1794649 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.s.IndexSchema [collection0_shard1_replica1] Schema name=minimal
   [junit4]   2> 1794660 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1794661 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'collection0_shard1_replica1' using configuration from collection collection0
   [junit4]   2> 1794662 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore [[collection0_shard1_replica1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/collection0_shard1_replica1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/./collection0_shard1_replica1/data/]
   [junit4]   2> 1794692 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1794701 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.IndexSchema [collection1_shard1_replica1] Schema name=minimal
   [junit4]   2> 1794703 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1794706 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica1' using configuration from collection collection1
   [junit4]   2> 1794706 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [[collection1_shard1_replica1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/collection1_shard1_replica1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/./collection1_shard1_replica1/data/]
   [junit4]   2> 1794708 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1794708 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1794711 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5afa98bf[collection0_shard1_replica1] main]
   [junit4]   2> 1794718 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1794718 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1794718 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@577b58e0[collection1_shard1_replica1] main]
   [junit4]   2> 1794720 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/testconfig
   [junit4]   2> 1794721 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/testconfig
   [junit4]   2> 1794721 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1794732 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1794732 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1794732 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:56041/solr/collection1_shard1_replica1/
   [junit4]   2> 1794732 ERROR (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SyncStrategy No UpdateLog found - cannot sync
   [junit4]   2> 1794732 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 1794736 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:56041/solr/collection1_shard1_replica1/ shard1
   [junit4]   2> 1794737 INFO  (searcherExecutor-2275-thread-1-processing-n:127.0.0.1:56041_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [collection1_shard1_replica1] Registered new searcher Searcher@577b58e0[collection1_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1794738 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/testconfig
   [junit4]   2> 1794738 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/testconfig
   [junit4]   2> 1794739 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1794739 INFO  (searcherExecutor-2273-thread-1-processing-n:127.0.0.1:56041_solr x:collection0_shard1_replica1 s:shard1 c:collection0 r:core_node1) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore [collection0_shard1_replica1] Registered new searcher Searcher@5afa98bf[collection0_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1794744 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1794748 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1794748 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1794748 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:56041/solr/collection0_shard1_replica1/
   [junit4]   2> 1794748 ERROR (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SyncStrategy No UpdateLog found - cannot sync
   [junit4]   2> 1794748 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 1794759 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:56041/solr/collection0_shard1_replica1/ shard1
   [junit4]   2> 1794787 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1794789 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=testconfig&name=collection1_shard1_replica1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2} status=0 QTime=1113
   [junit4]   2> 1794790 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1794797 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1794815 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1794821 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=testconfig&name=collection0_shard1_replica1&action=CREATE&numShards=1&collection=collection0&shard=shard1&wt=javabin&version=2} status=0 QTime=1249
   [junit4]   2> 1794822 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1794829 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1794926 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1795468 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000016 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1795468 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000018 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1795798 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=testconfig&name=collection1&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2343
   [junit4]   2> 1795799 INFO  (qtp1908042262-5757) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=collection1&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1795822 INFO  (OverseerThreadFactory-2254-thread-4-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 1795823 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr    ] o.a.s.c.SolrCore [collection1_shard1_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@7f736999
   [junit4]   2> 1795826 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=collection1_shard1_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=3
   [junit4]   2> 1795829 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=testconfig&name=collection0&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2526
   [junit4]   2> 1795830 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=collection0&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1795852 INFO  (OverseerThreadFactory-2254-thread-5-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 1795853 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr    ] o.a.s.c.SolrCore [collection0_shard1_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@639add30
   [junit4]   2> 1795860 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=collection0_shard1_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=7
   [junit4]   2> 1796444 INFO  (qtp1908042262-5757) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=collection1&action=DELETE&wt=javabin&version=2} status=0 QTime=645
   [junit4]   2> 1796445 INFO  (qtp1908042262-5757) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=testconfig&name=collection1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1796452 INFO  (OverseerThreadFactory-2254-thread-4-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 1796453 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000020 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1796573 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=collection0&action=DELETE&wt=javabin&version=2} status=0 QTime=742
   [junit4]   2> 1796574 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=testconfig&name=collection0&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1796576 INFO  (OverseerThreadFactory-2254-thread-5-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection0
   [junit4]   2> 1796577 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000022 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1796691 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=testconfig&name=collection1_shard1_replica1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1796893 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=testconfig&name=collection0_shard1_replica1&action=CREATE&numShards=1&collection=collection0&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1796896 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1797000 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1797753 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1797774 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.IndexSchema [collection1_shard1_replica1] Schema name=minimal
   [junit4]   2> 1797785 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1797786 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica1' using configuration from collection collection1
   [junit4]   2> 1797787 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [[collection1_shard1_replica1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/collection1_shard1_replica1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/./collection1_shard1_replica1/data/]
   [junit4]   2> 1797817 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1797817 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1797827 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@47882b23[collection1_shard1_replica1] main]
   [junit4]   2> 1797828 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/testconfig
   [junit4]   2> 1797828 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/testconfig
   [junit4]   2> 1797828 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1797835 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1797836 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1797836 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:56041/solr/collection1_shard1_replica1/
   [junit4]   2> 1797836 ERROR (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SyncStrategy No UpdateLog found - cannot sync
   [junit4]   2> 1797836 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 1797846 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:56041/solr/collection1_shard1_replica1/ shard1
   [junit4]   2> 1797848 INFO  (searcherExecutor-2281-thread-1-processing-n:127.0.0.1:56041_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [collection1_shard1_replica1] Registered new searcher Searcher@47882b23[collection1_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1797903 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1797915 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.s.IndexSchema [collection0_shard1_replica1] Schema name=minimal
   [junit4]   2> 1797921 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1797922 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'collection0_shard1_replica1' using configuration from collection collection0
   [junit4]   2> 1797923 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore [[collection0_shard1_replica1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/collection0_shard1_replica1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/./collection0_shard1_replica1/data/]
   [junit4]   2> 1797935 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1797936 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1797936 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6f39c0ba[collection0_shard1_replica1] main]
   [junit4]   2> 1797937 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/testconfig
   [junit4]   2> 1797938 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/testconfig
   [junit4]   2> 1797938 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1797943 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1797943 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1797943 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:56041/solr/collection0_shard1_replica1/
   [junit4]   2> 1797943 ERROR (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SyncStrategy No UpdateLog found - cannot sync
   [junit4]   2> 1797944 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 1797944 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1797945 INFO  (searcherExecutor-2285-thread-1-processing-n:127.0.0.1:56041_solr x:collection0_shard1_replica1 s:shard1 c:collection0 r:core_node1) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore [collection0_shard1_replica1] Registered new searcher Searcher@6f39c0ba[collection0_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1797949 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1797951 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:56041/solr/collection0_shard1_replica1/ shard1
   [junit4]   2> 1797961 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=testconfig&name=collection1_shard1_replica1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2} status=0 QTime=1270
   [junit4]   2> 1797964 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1797965 INFO  (qtp1908042262-5757) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1798007 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.ZkController I am the leader, no recovery necessary
   [junit4]   2> 1798013 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1798024 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={qt=/admin/cores&collection.configName=testconfig&name=collection0_shard1_replica1&action=CREATE&numShards=1&collection=collection0&shard=shard1&wt=javabin&version=2} status=0 QTime=1131
   [junit4]   2> 1798027 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Wait for new collection to be active for at most 30 seconds. Check all shard replicas
   [junit4]   2> 1798117 INFO  (zkCallback-805-thread-3-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1798581 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000024 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1798581 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000026 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1798966 INFO  (qtp1908042262-5757) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=testconfig&name=collection1&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2520
   [junit4]   2> 1798967 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=collection1&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1798969 INFO  (OverseerThreadFactory-2254-thread-5-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 1798970 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr    ] o.a.s.c.SolrCore [collection1_shard1_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@3f6c5682
   [junit4]   2> 1798975 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=collection1_shard1_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=5
   [junit4]   2> 1799027 INFO  (qtp1908042262-5755) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={replicationFactor=1&collection.configName=testconfig&name=collection0&action=CREATE&numShards=1&wt=javabin&version=2} status=0 QTime=2453
   [junit4]   2> 1799029 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :delete with params name=collection0&action=DELETE&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1799031 INFO  (OverseerThreadFactory-2254-thread-4-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerCollectionMessageHandler Executing Collection Cmd : action=UNLOAD&deleteInstanceDir=true&deleteDataDir=true
   [junit4]   2> 1799032 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr    ] o.a.s.c.SolrCore [collection0_shard1_replica1]  CLOSING SolrCore org.apache.solr.core.SolrCore@5aae268b
   [junit4]   2> 1799040 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/cores params={deleteInstanceDir=true&core=collection0_shard1_replica1&qt=/admin/cores&deleteDataDir=true&action=UNLOAD&wt=javabin&version=2} status=0 QTime=7
   [junit4]   2> 1799586 INFO  (qtp1908042262-5761) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=collection1&action=DELETE&wt=javabin&version=2} status=0 QTime=619
   [junit4]   2> 1799592 INFO  (qtp1908042262-5757) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=testconfig&name=collection1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1799594 INFO  (OverseerThreadFactory-2254-thread-5-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 1799595 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000028 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1799770 INFO  (qtp1908042262-5758) [n:127.0.0.1:56041_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={name=collection0&action=DELETE&wt=javabin&version=2} status=0 QTime=741
   [junit4]   2> 1799772 INFO  (qtp1908042262-5759) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=testconfig&name=collection0&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 1799784 INFO  (OverseerThreadFactory-2254-thread-4-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection0
   [junit4]   2> 1799785 INFO  (OverseerCollectionConfigSetProcessor-96708752676749315-127.0.0.1:56041_solr-n_0000000000) [n:127.0.0.1:56041_solr    ] o.a.s.c.OverseerTaskQueue Response ZK path: /overseer/collection-queue-work/qnr-0000000030 doesn't exist.  Requestor may have disconnected from ZooKeeper
   [junit4]   2> 1799822 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=testconfig&name=collection1_shard1_replica1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1799889 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=testconfig&name=collection0_shard1_replica1&action=CREATE&numShards=1&collection=collection0&shard=shard1&wt=javabin&version=2
   [junit4]   2> 1799893 INFO  (zkCallback-805-thread-1-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1799997 INFO  (zkCallback-805-thread-1-processing-n:127.0.0.1:56041_solr) [n:127.0.0.1:56041_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 1800833 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1800841 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.IndexSchema [collection1_shard1_replica1] Schema name=minimal
   [junit4]   2> 1800852 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1800854 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'collection1_shard1_replica1' using configuration from collection collection1
   [junit4]   2> 1800854 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [[collection1_shard1_replica1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/collection1_shard1_replica1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/./collection1_shard1_replica1/data/]
   [junit4]   2> 1800872 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1800872 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1800874 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3f73c646[collection1_shard1_replica1] main]
   [junit4]   2> 1800875 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/testconfig
   [junit4]   2> 1800876 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/testconfig
   [junit4]   2> 1800876 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1800886 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 1800886 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 1800886 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SyncStrategy Sync replicas to https://127.0.0.1:56041/solr/collection1_shard1_replica1/
   [junit4]   2> 1800886 ERROR (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SyncStrategy No UpdateLog found - cannot sync
   [junit4]   2> 1800886 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext We failed sync, but we have no versions - we can't sync in that case - we were active before, so become leader anyway
   [junit4]   2> 1800886 INFO  (searcherExecutor-2289-thread-1-processing-n:127.0.0.1:56041_solr x:collection1_shard1_replica1 s:shard1 c:collection1 r:core_node1) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.SolrCore [collection1_shard1_replica1] Registered new searcher Searcher@3f73c646[collection1_shard1_replica1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1800908 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1800962 INFO  (qtp1908042262-5754) [n:127.0.0.1:56041_solr c:collection1 s:shard1 r:core_node1 x:collection1_shard1_replica1] o.a.s.c.ShardLeaderElectionContext I am the new leader: https://127.0.0.1:56041/solr/collection1_shard1_replica1/ shard1
   [junit4]   2> 1800973 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.s.IndexSchema [collection0_shard1_replica1] Schema name=minimal
   [junit4]   2> 1800975 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.s.IndexSchema Loaded schema minimal/1.1 with uniqueid field id
   [junit4]   2> 1800976 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'collection0_shard1_replica1' using configuration from collection collection0
   [junit4]   2> 1800976 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore [[collection0_shard1_replica1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/collection0_shard1_replica1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_BF6C5C74A9CEFC3D-001/tempDir-001/node1/./collection0_shard1_replica1/data/]
   [junit4]   2> 1800987 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1800988 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1800988 INFO  (qtp1908042262-5760) [n:127.0.0.1:56041_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.s.SolrIndexSearcher Opening [Searcher@351b4b32[collection0_shard1_replica1] main]
   [junit4]   2> 1800989 I

[...truncated too long message...]

ntAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: java.lang.NullPointerException
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
   [junit4]   2> 	... 54 more
   [junit4]   2> 15655648 INFO  (SUITE-StressHdfsTest-seed#[BF6C5C74A9CEFC3D]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.StressHdfsTest_BF6C5C74A9CEFC3D-001
   [junit4]   2> Oct 05, 2016 12:58:47 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene70, sim=RandomSimilarity(queryNorm=false): {}, locale=zh-SG, timezone=Pacific/Honolulu
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=2,free=294931632,total=506462208
   [junit4]   2> NOTE: All tests run in this JVM: [ExplicitHLLTest, CoreSorterTest, PluginInfoTest, TestLegacyNumericRangeQueryBuilder, AtomicUpdatesTest, DistributedFacetPivotLongTailTest, TestGraphTermsQParserPlugin, BufferStoreTest, QueryElevationComponentTest, JavabinLoaderTest, TlogReplayBufferedWhileIndexingTest, RequestLoggingTest, QueryEqualityTest, TestRealTimeGet, CollectionTooManyReplicasTest, TestFieldTypeResource, TestManagedSchemaAPI, TestHighFrequencyDictionaryFactory, TestLockTree, TestClusterStateMutator, TestRangeQuery, BasicAuthIntegrationTest, SearchHandlerTest, RegexBoostProcessorTest, DeleteShardTest, TestStressLucene, TestConfigOverlay, BigEndianAscendingWordDeserializerTest, BasicDistributedZkTest, TestSolrCoreProperties, HdfsDirectoryFactoryTest, TestDFISimilarityFactory, TestHdfsCloudBackupRestore, SolrCoreTest, TestSortByMinMaxFunction, TestSubQueryTransformer, TestCustomSort, TestLeaderInitiatedRecoveryThread, SuggestComponentContextFilterQueryTest, TestRandomRequestDistribution, NumericFieldsTest, TestValueSourceCache, ResponseLogComponentTest, TestSchemaSimilarityResource, UniqFieldsUpdateProcessorFactoryTest, TestSolrFieldCacheMBean, TestBadConfig, TestFieldCollectionResource, TestExpandComponent, HdfsRecoverLeaseTest, TestShortCircuitedRequests, TestRequestForwarding, TestOmitPositions, TestSolrDeletionPolicy2, TestRTimerTree, SystemInfoHandlerTest, ReturnFieldsTest, SyncSliceTest, TestGraphMLResponseWriter, SolrCloudExampleTest, DisMaxRequestHandlerTest, TestAtomicUpdateErrorCases, DistributedSpellCheckComponentTest, TestLFUCache, BlockJoinFacetRandomTest, ConcurrentDeleteAndCreateCollectionTest, TestDocTermOrds, TestConfigSetsAPIExclusivity, RestartWhileUpdatingTest, TestCSVLoader, TestDistribDocBasedVersion, CreateCollectionCleanupTest, CursorPagingTest, ResponseHeaderTest, TestLRUCache, TestStressLiveNodes, NotRequiredUniqueKeyTest, DateMathParserTest, ClusterStateTest, BooleanFieldTest, RankQueryTest, PolyFieldTest, DocValuesMissingTest, ZkNodePropsTest, TestNumericTerms32, TestFieldCacheVsDocValues, TestLMJelinekMercerSimilarityFactory, FastVectorHighlighterTest, CoreMergeIndexesAdminHandlerTest, ScriptEngineTest, AnalyticsQueryTest, TestRemoteStreaming, AsyncCallRequestStatusResponseTest, TestHdfsBackupRestoreCore, DistributedFacetPivotWhiteBoxTest, TestDistributedSearch, TestManagedResource, TestSolrQueryResponse, TestManagedStopFilterFactory, TestMiniSolrCloudClusterSSL, SpellCheckCollatorTest, TestFoldingMultitermQuery, SuggesterWFSTTest, SolrCoreCheckLockOnStartupTest, TestPseudoReturnFields, DefaultValueUpdateProcessorTest, SortByFunctionTest, DocValuesMultiTest, DistanceFunctionTest, SolrInfoMBeanTest, XsltUpdateRequestHandlerTest, LukeRequestHandlerTest, TestQueryTypes, FileBasedSpellCheckerTest, PathHierarchyTokenizerFactoryTest, MoreLikeThisHandlerTest, IndexSchemaRuntimeFieldTest, IndexSchemaTest, TestCollationField, QueryParsingTest, TestAnalyzedSuggestions, TestPHPSerializedResponseWriter, CSVRequestHandlerTest, SOLR749Test, TestQuerySenderListener, TestQuerySenderNoQuery, CopyFieldTest, SolrIndexConfigTest, BadComponentTest, TestStressRecovery, EchoParamsTest, TestLMDirichletSimilarityFactory, TestFastWriter, ResourceLoaderTest, TestFastOutputStream, ChaosMonkeyNothingIsSafeTest, AssignTest, CdcrBootstrapTest, CdcrRequestHandlerTest, CdcrVersionReplicationTest, CleanupOldIndexTest, CloudExitableDirectoryReaderTest, CollectionStateFormat2Test, ConnectionManagerTest, CustomCollectionTest, DeleteLastCustomShardedReplicaTest, DeleteStatusTest, DistribDocExpirationUpdateProcessorTest, DistribJoinFromCollectionTest, DistributedQueueTest, DistributedVersionInfoTest, ForceLeaderTest, HttpPartitionTest, LeaderFailoverAfterPartitionTest, LeaderFailureAfterFreshStartTest, LeaderInitiatedRecoveryOnShardRestartTest, MigrateRouteKeyTest, MultiThreadedOCPTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerModifyCollectionTest, OverseerRolesTest, OverseerStatusTest, OverseerTaskQueueTest, PeerSyncReplicationTest, RecoveryAfterSoftCommitTest, RemoteQueryErrorTest, ReplaceNodeTest, ReplicationFactorTest, SSLMigrationTest, SaslZkACLProviderTest, SharedFSAutoReplicaFailoverTest, SharedFSAutoReplicaFailoverUtilsTest, SolrCLIZkUtilsTest, SolrXmlInZkTest, TestAuthenticationFramework, TestCloudDeleteByQuery, TestCloudInspectUtil, TestCloudPseudoReturnFields, TestConfigSetsAPI, TestExclusionRuleCollectionAccess, TestLocalFSCloudBackupRestore, TestRandomFlRTGCloud, TestRequestStatusCollectionAPI, TestSSLRandomization, TestSolrCloudWithKerberosAlt, TestStressCloudBlindAtomicUpdates, TriLevelCompositeIdRoutingTest, VMParamsZkACLAndCredentialsProvidersTest, HdfsBasicDistributedZk2Test, HdfsBasicDistributedZkTest, HdfsChaosMonkeyNothingIsSafeTest, HdfsChaosMonkeySafeLeaderTest, HdfsNNFailoverTest, HdfsSyncSliceTest, HdfsThreadLeakTest, HdfsUnloadDistributedZkTest, StressHdfsTest]
   [junit4] Completed [428/635 (6!)] on J0 in 88.19s, 1 test, 1 failure <<< FAILURES!

[...truncated 420 lines...]
   [junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20161005_083740_204.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid6781.hprof ...
   [junit4] Heap dump file created [588108079 bytes in 19.831 secs]
   [junit4] <<< JVM J0: EOF ----

[...truncated 7400 lines...]
ERROR: Connection was broken: java.io.IOException: Unexpected termination of the channel
	at hudson.remoting.SynchronousCommandTransport$ReaderThread.run(SynchronousCommandTransport.java:73)
Caused by: java.io.EOFException
	at java.io.ObjectInputStream$PeekInputStream.readFully(ObjectInputStream.java:2328)
	at java.io.ObjectInputStream$BlockDataInputStream.readShort(ObjectInputStream.java:2797)
	at java.io.ObjectInputStream.readStreamHeader(ObjectInputStream.java:802)
	at java.io.ObjectInputStream.<init>(ObjectInputStream.java:299)
	at hudson.remoting.ObjectInputStreamEx.<init>(ObjectInputStreamEx.java:48)
	at hudson.remoting.AbstractSynchronousByteArrayCommandTransport.read(AbstractSynchronousByteArrayCommandTransport.java:34)
	at hudson.remoting.SynchronousCommandTransport$ReaderThread.run(SynchronousCommandTransport.java:59)

Build step 'Invoke Ant' marked build as failure
ERROR: Step ‘Archive the artifacts’ failed: no workspace for Lucene-Solr-NightlyTests-master #1126
ERROR: Step ‘Publish JUnit test result report’ failed: no workspace for Lucene-Solr-NightlyTests-master #1126
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1125 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1125/

13 tests failed.
FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBufferOnNonLeader

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([AD179BA835260260]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CdcrReplicationDistributedZkTest

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([AD179BA835260260]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CdcrReplicationDistributedZkTest

Error Message:
Captured an uncaught exception in thread: Thread[id=125282, name=zkCallback-29018-thread-8, state=RUNNABLE, group=TGRP-CdcrReplicationDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=125282, name=zkCallback-29018-thread-8, state=RUNNABLE, group=TGRP-CdcrReplicationDistributedZkTest]
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
	at __randomizedtesting.SeedInfo.seed([AD179BA835260260]:0)
	at java.lang.ThreadLocal.createMap(ThreadLocal.java:244)
	at java.lang.ThreadLocal.set(ThreadLocal.java:205)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:215)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$9/79255960.run(Unknown Source)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CdcrReplicationDistributedZkTest

Error Message:
Captured an uncaught exception in thread: Thread[id=125278, name=zkCallback-29018-thread-4, state=RUNNABLE, group=TGRP-CdcrReplicationDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=125278, name=zkCallback-29018-thread-4, state=RUNNABLE, group=TGRP-CdcrReplicationDistributedZkTest]
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
	at __randomizedtesting.SeedInfo.seed([AD179BA835260260]:0)
	at java.lang.ThreadLocal.createMap(ThreadLocal.java:244)
	at java.lang.ThreadLocal.set(ThreadLocal.java:205)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:215)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$9/79255960.run(Unknown Source)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CdcrReplicationDistributedZkTest

Error Message:
Captured an uncaught exception in thread: Thread[id=121612, name=Connection evictor, state=RUNNABLE, group=TGRP-CdcrReplicationDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=121612, name=Connection evictor, state=RUNNABLE, group=TGRP-CdcrReplicationDistributedZkTest]
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.cloud.CdcrReplicationHandlerTest.testFullReplication

Error Message:
There are still nodes recoverying - waited for 330 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 330 seconds
	at __randomizedtesting.SeedInfo.seed([AD179BA835260260:D717A1E5629944E0]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:181)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:138)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:133)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForRecoveriesToFinish(BaseCdcrDistributedZkTest.java:509)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.restartServer(BaseCdcrDistributedZkTest.java:550)
	at org.apache.solr.cloud.CdcrReplicationHandlerTest.testFullReplication(CdcrReplicationHandlerTest.java:82)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrRequestHandlerTest.testBufferActions

Error Message:
Captured an uncaught exception in thread: Thread[id=83479, name=qtp428698500-83479, state=RUNNABLE, group=TGRP-CdcrRequestHandlerTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=83479, name=qtp428698500-83479, state=RUNNABLE, group=TGRP-CdcrRequestHandlerTest]
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
	at __randomizedtesting.SeedInfo.seed([AD179BA835260260]:0)
	at java.lang.Thread.start0(Native Method)
	at java.lang.Thread.start(Thread.java:714)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.startThreads(QueuedThreadPool.java:462)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.access$200(QueuedThreadPool.java:47)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:639)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
Timeout waiting for all live and active

Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
	at __randomizedtesting.SeedInfo.seed([AD179BA835260260:2543A4729BDA6F98]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:309)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:127)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
Expected 2 of 3 replicas to be active but only found 1; [core_node2:{"core":"c8n_1x3_lf_shard1_replica3","base_url":"http://127.0.0.1:51520/_uzc","node_name":"127.0.0.1:51520__uzc","state":"active","leader":"true"}]; clusterState: DocCollection(c8n_1x3_lf//collections/c8n_1x3_lf/state.json/19)={   "replicationFactor":"3",   "shards":{"shard1":{       "range":"80000000-7fffffff",       "state":"active",       "replicas":{         "core_node1":{           "state":"down",           "base_url":"http://127.0.0.1:45032/_uzc",           "core":"c8n_1x3_lf_shard1_replica1",           "node_name":"127.0.0.1:45032__uzc"},         "core_node2":{           "core":"c8n_1x3_lf_shard1_replica3",           "base_url":"http://127.0.0.1:51520/_uzc",           "node_name":"127.0.0.1:51520__uzc",           "state":"active",           "leader":"true"},         "core_node3":{           "core":"c8n_1x3_lf_shard1_replica2",           "base_url":"http://127.0.0.1:34773/_uzc",           "node_name":"127.0.0.1:34773__uzc",           "state":"down"}}}},   "router":{"name":"compositeId"},   "maxShardsPerNode":"1",   "autoAddReplicas":"false"}

Stack Trace:
java.lang.AssertionError: Expected 2 of 3 replicas to be active but only found 1; [core_node2:{"core":"c8n_1x3_lf_shard1_replica3","base_url":"http://127.0.0.1:51520/_uzc","node_name":"127.0.0.1:51520__uzc","state":"active","leader":"true"}]; clusterState: DocCollection(c8n_1x3_lf//collections/c8n_1x3_lf/state.json/19)={
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node1":{
          "state":"down",
          "base_url":"http://127.0.0.1:45032/_uzc",
          "core":"c8n_1x3_lf_shard1_replica1",
          "node_name":"127.0.0.1:45032__uzc"},
        "core_node2":{
          "core":"c8n_1x3_lf_shard1_replica3",
          "base_url":"http://127.0.0.1:51520/_uzc",
          "node_name":"127.0.0.1:51520__uzc",
          "state":"active",
          "leader":"true"},
        "core_node3":{
          "core":"c8n_1x3_lf_shard1_replica2",
          "base_url":"http://127.0.0.1:34773/_uzc",
          "node_name":"127.0.0.1:34773__uzc",
          "state":"down"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false"}
	at __randomizedtesting.SeedInfo.seed([AD179BA835260260:2543A4729BDA6F98]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:170)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:57)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)
	at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2251)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)
	at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2254)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(Sys

[...truncated too long message...]

e.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 12100615 WARN  (TEST-CdcrReplicationDistributedZkTest.testBufferOnNonLeader-seed#[AD179BA835260260]) [    ] o.a.s.c.ZkController Could not publish node as down: java.lang.InterruptedException: sleep interrupted
   [junit4]   2> 12100616 INFO  (TEST-CdcrReplicationDistributedZkTest.testBufferOnNonLeader-seed#[AD179BA835260260]) [    ] o.a.s.c.Overseer Overseer (id=96700331920916505-127.0.0.1:53309_snxqv-n_0000000006) closing
   [junit4]   2> 12100674 INFO  (TEST-CdcrReplicationDistributedZkTest.testBufferOnNonLeader-seed#[AD179BA835260260]) [    ] o.e.j.s.ServerConnector Stopped ServerConnector@45d0ea17{SSL,[ssl, http/1.1]}{127.0.0.1:0}
   [junit4]   2> 12100674 INFO  (TEST-CdcrReplicationDistributedZkTest.testBufferOnNonLeader-seed#[AD179BA835260260]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@4158b3bb{/snxqv,null,UNAVAILABLE}
   [junit4]   2> 12100786 INFO  (TEST-CdcrReplicationDistributedZkTest.testBufferOnNonLeader-seed#[AD179BA835260260]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testBufferOnNonLeader
   [junit4]   2> 12100788 INFO  (TEST-CdcrReplicationDistributedZkTest.testBufferOnNonLeader-seed#[AD179BA835260260]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:60345 60345
   [junit4]   2> 12101120 INFO  (SUITE-CdcrReplicationDistributedZkTest-seed#[AD179BA835260260]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_AD179BA835260260-001
   [junit4]   2> Oct 03, 2016 11:28:07 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {}, docValues:{}, maxPointsInLeafNode=990, maxMBSortInHeap=7.464792411998001, sim=RandomSimilarity(queryNorm=false): {}, locale=sr-Latn-RS, timezone=Pacific/Kwajalein
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=2,free=117083504,total=493879296
   [junit4]   2> NOTE: All tests run in this JVM: [CollectionsAPIDistributedZkTest, BlockJoinFacetRandomTest, TestCoreContainer, TestFiltering, TestElisionMultitermQuery, ResponseHeaderTest, TestRandomRequestDistribution, TestExpandComponent, TestStressLiveNodes, TestSchemaSimilarityResource, TestSimpleQParserPlugin, CSVRequestHandlerTest, TestSimpleTrackingShardHandler, TestSolrCloudWithKerberosAlt, TestJsonFacets, FileUtilsTest, TestLeaderInitiatedRecoveryThread, ZkControllerTest, AlternateDirectoryTest, TestBinaryResponseWriter, SharedFSAutoReplicaFailoverUtilsTest, TestCloudDeleteByQuery, TestSolrQueryParserDefaultOperatorResource, SOLR749Test, PreAnalyzedUpdateProcessorTest, CoreMergeIndexesAdminHandlerTest, TestBulkSchemaConcurrent, TestNumericTerms32, TestDocTermOrds, MoreLikeThisHandlerTest, HdfsDirectoryFactoryTest, DistanceFunctionTest, SaslZkACLProviderTest, TestConfigSets, ClusterStateUpdateTest, TestShortCircuitedRequests, ConvertedLegacyTest, RestartWhileUpdatingTest, CollectionTooManyReplicasTest, DistributedIntervalFacetingTest, TestMiniSolrCloudClusterSSL, TermVectorComponentTest, BasicFunctionalityTest, CreateCollectionCleanupTest, SpellingQueryConverterTest, LeaderInitiatedRecoveryOnShardRestartTest, TestBM25SimilarityFactory, TestLockTree, IndexSchemaRuntimeFieldTest, AliasIntegrationTest, TestSolrFieldCacheMBean, TestDistribDocBasedVersion, IndexSchemaTest, AddBlockUpdateTest, ConcurrentDeleteAndCreateCollectionTest, TestHdfsBackupRestoreCore, TestDefaultSearchFieldResource, TestUpdate, LeaderFailoverAfterPartitionTest, SpatialFilterTest, TestManagedResource, RequestLoggingTest, TestMacros, UUIDUpdateProcessorFallbackTest, TestStressRecovery, SpellCheckCollatorTest, DirectSolrConnectionTest, DirectSolrSpellCheckerTest, TestHdfsUpdateLog, TestHdfsCloudBackupRestore, IndexBasedSpellCheckerTest, TestDistributedGrouping, HdfsUnloadDistributedZkTest, TestScoreJoinQPNoScore, HdfsChaosMonkeySafeLeaderTest, DistribDocExpirationUpdateProcessorTest, OpenCloseCoreStressTest, SolrIndexSplitterTest, OpenExchangeRatesOrgProviderTest, DocExpirationUpdateProcessorFactoryTest, TestRandomCollapseQParserPlugin, TlogReplayBufferedWhileIndexingTest, TestCursorMarkWithoutUniqueKey, JavabinLoaderTest, TestSolrConfigHandlerConcurrent, TestPostingsSolrHighlighter, SortByFunctionTest, SampleTest, TestFuzzyAnalyzedSuggestions, TestQueryUtils, OverseerTaskQueueTest, TestFieldCacheVsDocValues, QueryParsingTest, DocValuesMissingTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestFieldTypeResource, TestSolrDynamicMBean, TestCustomSort, TestFieldCacheSanityChecker, BigEndianAscendingWordDeserializerTest, SolrCmdDistributorTest, OverseerModifyCollectionTest, TestTrie, CdcrReplicationDistributedZkTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=CdcrReplicationDistributedZkTest -Dtests.seed=AD179BA835260260 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=sr-Latn-RS -Dtests.timezone=Pacific/Kwajalein -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J1 | CdcrReplicationDistributedZkTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AD179BA835260260]:0)Throwable #2: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=125282, name=zkCallback-29018-thread-8, state=RUNNABLE, group=TGRP-CdcrReplicationDistributedZkTest]
   [junit4]    > Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AD179BA835260260]:0)
   [junit4]    > 	at java.lang.ThreadLocal.createMap(ThreadLocal.java:244)
   [junit4]    > 	at java.lang.ThreadLocal.set(ThreadLocal.java:205)
   [junit4]    > 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:215)
   [junit4]    > 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$9/79255960.run(Unknown Source)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #3: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=125278, name=zkCallback-29018-thread-4, state=RUNNABLE, group=TGRP-CdcrReplicationDistributedZkTest]
   [junit4]    > Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AD179BA835260260]:0)
   [junit4]    > 	at java.lang.ThreadLocal.createMap(ThreadLocal.java:244)
   [junit4]    > 	at java.lang.ThreadLocal.set(ThreadLocal.java:205)
   [junit4]    > 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:215)
   [junit4]    > 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor$$Lambda$9/79255960.run(Unknown Source)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #4: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=121612, name=Connection evictor, state=RUNNABLE, group=TGRP-CdcrReplicationDistributedZkTest]
   [junit4]    > Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Completed [636/636 (8!)] on J1 in 7320.25s, 1 test, 5 errors <<< FAILURES!

[...truncated 1 lines...]
   [junit4] JVM J1: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20161003_200617_507.sysout
   [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid10181.hprof ...
   [junit4] Heap dump file created [589912235 bytes in 13.697 secs]
   [junit4] <<< JVM J1: EOF ----

[...truncated 11039 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:715: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid10181.hprof

Total time: 507 minutes 52 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1124 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1124/

13 tests failed.
FAILED:  org.apache.solr.cloud.CdcrBootstrapTest.testConvertClusterToCdcrAndBootstrap

Error Message:
Document mismatch on target after sync expected:<10000> but was:<0>

Stack Trace:
java.lang.AssertionError: Document mismatch on target after sync expected:<10000> but was:<0>
	at __randomizedtesting.SeedInfo.seed([49AB02AE84B67800:9E7C2DD930E9E047]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.apache.solr.cloud.CdcrBootstrapTest.testConvertClusterToCdcrAndBootstrap(CdcrBootstrapTest.java:144)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationHandlerTest.testReplicationWithBufferedUpdates

Error Message:
Timeout while trying to assert number of documents @ source_collection

Stack Trace:
java.lang.AssertionError: Timeout while trying to assert number of documents @ source_collection
	at __randomizedtesting.SeedInfo.seed([49AB02AE84B67800:9AA252B0C125E497]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:273)
	at org.apache.solr.cloud.CdcrReplicationHandlerTest.testReplicationWithBufferedUpdates(CdcrReplicationHandlerTest.java:233)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.AssertionError: expected:<570> but was:<580>
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:264)
	... 42 more


FAILED:  org.apache.solr.cloud.CdcrReplicationHandlerTest.testPartialReplication

Error Message:
unable to create new native thread

Stack Trace:
java.lang.OutOfMemoryError: unable to create new native thread
	at __randomizedtesting.SeedInfo.seed([49AB02AE84B67800:D789231EAC1EF86C]:0)
	at java.lang.Thread.start0(Native Method)
	at java.lang.Thread.start(Thread.java:714)
	at org.apache.http.impl.client.IdleConnectionEvictor.start(IdleConnectionEvictor.java:96)
	at org.apache.http.impl.client.HttpClientBuilder.build(HttpClientBuilder.java:1178)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:260)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.<init>(CloudSolrClient.java:387)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.<init>(CloudSolrClient.java:100)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$Builder.build(CloudSolrClient.java:1655)
	at org.apache.solr.SolrTestCaseJ4$CloudSolrClientBuilder.build(SolrTestCaseJ4.java:2115)
	at org.apache.solr.SolrTestCaseJ4.getCloudSolrClient(SolrTestCaseJ4.java:2147)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.createCloudClient(BaseCdcrDistributedZkTest.java:183)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForRecoveriesToFinish(BaseCdcrDistributedZkTest.java:505)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.restartServer(BaseCdcrDistributedZkTest.java:550)
	at org.apache.solr.cloud.CdcrReplicationHandlerTest.testPartialReplication(CdcrReplicationHandlerTest.java:116)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrRequestHandlerTest.testLifeCycleActions

Error Message:
Captured an uncaught exception in thread: Thread[id=67786, name=qtp1013051176-67786, state=RUNNABLE, group=TGRP-CdcrRequestHandlerTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=67786, name=qtp1013051176-67786, state=RUNNABLE, group=TGRP-CdcrRequestHandlerTest]
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
	at __randomizedtesting.SeedInfo.seed([49AB02AE84B67800]:0)
	at java.lang.Thread.start0(Native Method)
	at java.lang.Thread.start(Thread.java:714)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.startThreads(QueuedThreadPool.java:462)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.access$200(QueuedThreadPool.java:47)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:639)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CollectionStateFormat2Test.test

Error Message:
Shards in the state does not match what we set:3 vs 4

Stack Trace:
java.lang.AssertionError: Shards in the state does not match what we set:3 vs 4
	at __randomizedtesting.SeedInfo.seed([49AB02AE84B67800:C1FF3D742A4A15F8]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:419)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:332)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:983)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
Expected 2 of 3 replicas to be active but only found 1; [core_node3:{"core":"c8n_1x3_lf_shard1_replica2","base_url":"http://127.0.0.1:54428","node_name":"127.0.0.1:54428_","state":"active","leader":"true"}]; clusterState: DocCollection(c8n_1x3_lf//collections/c8n_1x3_lf/state.json/19)={   "replicationFactor":"3",   "shards":{"shard1":{       "range":"80000000-7fffffff",       "state":"active",       "replicas":{         "core_node1":{           "state":"down",           "base_url":"http://127.0.0.1:56560",           "core":"c8n_1x3_lf_shard1_replica1",           "node_name":"127.0.0.1:56560_"},         "core_node2":{           "core":"c8n_1x3_lf_shard1_replica3",           "base_url":"http://127.0.0.1:48001",           "node_name":"127.0.0.1:48001_",           "state":"down"},         "core_node3":{           "core":"c8n_1x3_lf_shard1_replica2",           "base_url":"http://127.0.0.1:54428",           "node_name":"127.0.0.1:54428_",           "state":"active",           "leader":"true"}}}},   "router":{"name":"compositeId"},   "maxShardsPerNode":"1",   "autoAddReplicas":"false"}

Stack Trace:
java.lang.AssertionError: Expected 2 of 3 replicas to be active but only found 1; [core_node3:{"core":"c8n_1x3_lf_shard1_replica2","base_url":"http://127.0.0.1:54428","node_name":"127.0.0.1:54428_","state":"active","leader":"true"}]; clusterState: DocCollection(c8n_1x3_lf//collections/c8n_1x3_lf/state.json/19)={
  "replicationFactor":"3",
  "shards":{"shard1":{
      "range":"80000000-7fffffff",
      "state":"active",
      "replicas":{
        "core_node1":{
          "state":"down",
          "base_url":"http://127.0.0.1:56560",
          "core":"c8n_1x3_lf_shard1_replica1",
          "node_name":"127.0.0.1:56560_"},
        "core_node2":{
          "core":"c8n_1x3_lf_shard1_replica3",
          "base_url":"http://127.0.0.1:48001",
          "node_name":"127.0.0.1:48001_",
          "state":"down"},
        "core_node3":{
          "core":"c8n_1x3_lf_shard1_replica2",
          "base_url":"http://127.0.0.1:54428",
          "node_name":"127.0.0.1:54428_",
          "state":"active",
          "leader":"true"}}}},
  "router":{"name":"compositeId"},
  "maxShardsPerNode":"1",
  "autoAddReplicas":"false"}
	at __randomizedtesting.SeedInfo.seed([49AB02AE84B67800:C1FF3D742A4A15F8]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:170)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:57)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.update.PeerSyncTest.test

Error Message:
.response[0][id][0]:3!=7

Stack Trace:
junit.framework.AssertionFailedError: .response[0][id][0]:3!=7
	at __randomizedtesting.SeedInfo.seed([49AB02AE84B67800:C1FF3D742A4A15F8]:0)
	at junit.framework.Assert.fail(Assert.java:50)
	at org.apache.solr.BaseDistributedSearchTestCase.compareSolrResponses(BaseDistributedSearchTestCase.java:913)
	at org.apache.solr.BaseDistributedSearchTestCase.compareResponses(BaseDistributedSearchTestCase.java:932)
	at org.apache.solr.BaseDistributedSearchTestCase.queryAndCompare(BaseDistributedSearchTestCase.java:650)
	at org.apache.solr.BaseDistributedSearchTestCase.queryAndCompare(BaseDistributedSearchTestCase.java:641)
	at org.apache.solr.update.PeerSyncTest.test(PeerSyncTest.java:97)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2250)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2250)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2250)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2250)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomize

[...truncated too long message...]

2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-collective-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-event-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/bcprov-jdk15-1.45.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/cglib-nodep-2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/commons-math3-3.4.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/easymock-3.0.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-common-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-hdfs-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-minikdc-2.7.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-annotations-2.5.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-databind-2.5.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/mockito-core-1.9.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/objenesis-1.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/presto-parser-0.122.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/slice-0.10.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/analysis/icu/lucene-analyzers-icu-7.0.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/analysis-extras/lib/icu4j-56.1.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_102/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.3.4.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20161002_094651_442.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20161002_094651_442.suites -stdin
   [junit4] ERROR: JVM J2 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1501)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:121)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:955)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:952)
   [junit4] 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4] 	at java.lang.Thread.run(Thread.java:745)

BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:707: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build.xml:252: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/common-build.xml:536: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:1443: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:984: At least one slave process threw an exception, first: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.

Total time: 429 minutes 5 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1123 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1123/

16 tests failed.
FAILED:  org.apache.lucene.search.TestFuzzyQuery.testRandom

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([2D27C75D3AAA0A9]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.search.TestFuzzyQuery

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([2D27C75D3AAA0A9]:0)


FAILED:  org.apache.solr.cloud.CdcrRequestHandlerTest.testBufferActions

Error Message:
Captured an uncaught exception in thread: Thread[id=68967, name=SessionTracker, state=RUNNABLE, group=TGRP-CdcrRequestHandlerTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=68967, name=SessionTracker, state=RUNNABLE, group=TGRP-CdcrRequestHandlerTest]
	at __randomizedtesting.SeedInfo.seed([DF3C5BBDBF828ECA:4FEA04327916FAC0]:0)
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
	at __randomizedtesting.SeedInfo.seed([DF3C5BBDBF828ECA]:0)
	at java.lang.Long.valueOf(Long.java:840)
	at org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:150)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CdcrVersionReplicationTest

Error Message:
8 threads leaked from SUITE scope at org.apache.solr.cloud.CdcrVersionReplicationTest:     1) Thread[id=236749, name=qtp1046926853-236749, state=TIMED_WAITING, group=TGRP-CdcrVersionReplicationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=236751, name=qtp1046926853-236751-acceptor-0@1ece7e42-ServerConnector@607df54b{SSL,[ssl, http/1.1]}{127.0.0.1:54464}, state=RUNNABLE, group=TGRP-CdcrVersionReplicationTest]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:382)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:593)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=236754, name=qtp1046926853-236754, state=TIMED_WAITING, group=TGRP-CdcrVersionReplicationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=236753, name=qtp1046926853-236753, state=TIMED_WAITING, group=TGRP-CdcrVersionReplicationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=236750, name=qtp1046926853-236750, state=RUNNABLE, group=TGRP-CdcrVersionReplicationTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=236755, name=qtp1046926853-236755, state=TIMED_WAITING, group=TGRP-CdcrVersionReplicationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=236752, name=qtp1046926853-236752, state=TIMED_WAITING, group=TGRP-CdcrVersionReplicationTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=236748, name=qtp1046926853-236748, state=RUNNABLE, group=TGRP-CdcrVersionReplicationTest]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 8 threads leaked from SUITE scope at org.apache.solr.cloud.CdcrVersionReplicationTest: 
   1) Thread[id=236749, name=qtp1046926853-236749, state=TIMED_WAITING, group=TGRP-CdcrVersionReplicationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=236751, name=qtp1046926853-236751-acceptor-0@1ece7e42-ServerConnector@607df54b{SSL,[ssl, http/1.1]}{127.0.0.1:54464}, state=RUNNABLE, group=TGRP-CdcrVersionReplicationTest]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:382)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:593)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=236754, name=qtp1046926853-236754, state=TIMED_WAITING, group=TGRP-CdcrVersionReplicationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=236753, name=qtp1046926853-236753, state=TIMED_WAITING, group=TGRP-CdcrVersionReplicationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=236750, name=qtp1046926853-236750, state=RUNNABLE, group=TGRP-CdcrVersionReplicationTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=236755, name=qtp1046926853-236755, state=TIMED_WAITING, group=TGRP-CdcrVersionReplicationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=236752, name=qtp1046926853-236752, state=TIMED_WAITING, group=TGRP-CdcrVersionReplicationTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=236748, name=qtp1046926853-236748, state=RUNNABLE, group=TGRP-CdcrVersionReplicationTest]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([DF3C5BBDBF828ECA]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CdcrVersionReplicationTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=236750, name=qtp1046926853-236750, state=RUNNABLE, group=TGRP-CdcrVersionReplicationTest]         at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)         at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)         at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)         at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)         at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=236748, name=qtp1046926853-236748, state=RUNNABLE, group=TGRP-CdcrVersionReplicationTest]         at sun.nio.ch.IOUtil.drain(Native Method)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=236750, name=qtp1046926853-236750, state=RUNNABLE, group=TGRP-CdcrVersionReplicationTest]
        at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method)
        at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317)
        at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207)
        at java.nio.channels.spi.AbstractSelector$1.interrupt(AbstractSelector.java:213)
        at java.nio.channels.spi.AbstractSelector.begin(AbstractSelector.java:219)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:92)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=236748, name=qtp1046926853-236748, state=RUNNABLE, group=TGRP-CdcrVersionReplicationTest]
        at sun.nio.ch.IOUtil.drain(Native Method)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([DF3C5BBDBF828ECA]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient] org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2250)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2250)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2250)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2250)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException  at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)  at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)  at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)  at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)  at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2250)  at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)  at java.lang.reflect.Method.invoke(Method.java:498)  at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)  at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)  at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)  at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)  at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)  at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)  at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)  at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)  at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)  at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)  at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)  at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)  at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)  at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)  at java.lang.Thread.run(Thread.java:745)  

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:182)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:166)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2247)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)
	at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2250)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)
	at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2250)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)

org.apache.solr.common.util.ObjectReleaseTracker$ObjectTrackerException
	at org.apache.solr.common.util.ObjectReleaseTracker.track(ObjectReleaseTracker.java:43)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:262)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:209)
	at org.apache.solr.client.solrj.impl.HttpClientUtil.createClient(HttpClientUtil.java:201)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.<init>(HttpSolrClient.java:209)
	at org.apache.solr.client.solrj.impl.HttpSolrClient$Builder.build(HttpSolrClient.java:847)
	at org.apache.solr.SolrTestCaseJ4.getHttpSolrClient(SolrTestCaseJ4.java:2250)
	at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail(ConcurrentDeleteAndCreateCollectionTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMar

[...truncated too long message...]

artStop-seed#[DF3C5BBDBF828ECA]) [    ] o.e.j.s.ServerConnector Stopped ServerConnector@6c2e8e32{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 7902385 INFO  (TEST-CdcrReplicationDistributedZkTest.testReplicationStartStop-seed#[DF3C5BBDBF828ECA]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@87752fc{/_vd/y,null,UNAVAILABLE}
   [junit4]   2> 7902386 INFO  (TEST-CdcrReplicationDistributedZkTest.testReplicationStartStop-seed#[DF3C5BBDBF828ECA]) [    ] o.a.s.c.ChaosMonkey monkey: stop shard! 57363
   [junit4]   2> 7902386 INFO  (TEST-CdcrReplicationDistributedZkTest.testReplicationStartStop-seed#[DF3C5BBDBF828ECA]) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=1872502112
   [junit4]   2> 7902392 INFO  (TEST-CdcrReplicationDistributedZkTest.testReplicationStartStop-seed#[DF3C5BBDBF828ECA]) [    ] o.a.s.c.Overseer Overseer (id=96685160968945696-127.0.0.1:57363__vd%2Fy-n_0000000007) closing
   [junit4]   2> 7902392 INFO  (OverseerStateUpdate-96685160968945696-127.0.0.1:57363__vd%2Fy-n_0000000007) [n:127.0.0.1:57363__vd%2Fy    ] o.a.s.c.Overseer Overseer Loop exiting : 127.0.0.1:57363__vd%2Fy
   [junit4]   2> 7902405 WARN  (zkCallback-3673-thread-3-processing-n:127.0.0.1:57363__vd%2Fy) [n:127.0.0.1:57363__vd%2Fy    ] o.a.s.c.c.ZkStateReader ZooKeeper watch triggered, but Solr cannot talk to ZK: [KeeperErrorCode = Session expired for /live_nodes]
   [junit4]   2> 7903480 WARN  (zkCallback-3673-thread-4-processing-n:127.0.0.1:57363__vd%2Fy) [n:127.0.0.1:57363__vd%2Fy c:target_collection s:shard2 r:core_node4 x:target_collection_shard2_replica2] o.a.s.c.SyncStrategy Closed, skipping sync up.
   [junit4]   2> 7903481 INFO  (zkCallback-3673-thread-4-processing-n:127.0.0.1:57363__vd%2Fy) [n:127.0.0.1:57363__vd%2Fy c:target_collection s:shard2 r:core_node4 x:target_collection_shard2_replica2] o.a.s.c.SolrCore [target_collection_shard2_replica2]  CLOSING SolrCore org.apache.solr.core.SolrCore@3676c76b
   [junit4]   2> 7903481 INFO  (zkCallback-3673-thread-4-processing-n:127.0.0.1:57363__vd%2Fy) [n:127.0.0.1:57363__vd%2Fy c:target_collection s:shard2 r:core_node4 x:target_collection_shard2_replica2] o.a.s.h.CdcrRequestHandler Solr core is being closed - shutting down CDCR handler @ target_collection:shard2
   [junit4]   2> 7903507 INFO  (TEST-CdcrReplicationDistributedZkTest.testReplicationStartStop-seed#[DF3C5BBDBF828ECA]) [    ] o.e.j.s.ServerConnector Stopped ServerConnector@6af6b1c5{HTTP/1.1,[http/1.1]}{127.0.0.1:0}
   [junit4]   2> 7903508 INFO  (TEST-CdcrReplicationDistributedZkTest.testReplicationStartStop-seed#[DF3C5BBDBF828ECA]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@7a326319{/_vd/y,null,UNAVAILABLE}
   [junit4]   2> 7903508 INFO  (TEST-CdcrReplicationDistributedZkTest.testReplicationStartStop-seed#[DF3C5BBDBF828ECA]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testReplicationStartStop
   [junit4]   2> 7903512 INFO  (TEST-CdcrReplicationDistributedZkTest.testReplicationStartStop-seed#[DF3C5BBDBF828ECA]) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:59905 59905
   [junit4]   2> 7903649 INFO  (Thread-20183) [    ] o.a.s.c.ZkTestServer connecting to 127.0.0.1:59905 59905
   [junit4]   2> 7903658 WARN  (Thread-20183) [    ] o.a.s.c.ZkTestServer Watch limit violations: 
   [junit4]   2> Maximum concurrent create/delete watches above limit:
   [junit4]   2> 
   [junit4]   2> 	33	/solr/clusterprops.json
   [junit4]   2> 	33	/solr/aliases.json
   [junit4]   2> 	16	/solr/configs/conf1
   [junit4]   2> 	8	/solr/collections/tmp_collection/state.json
   [junit4]   2> 	8	/solr/security.json
   [junit4]   2> 	3	/solr/collections/target_collection/leaders/shard2/leader
   [junit4]   2> 	3	/solr/collections/target_collection/leaders/shard1/leader
   [junit4]   2> 	2	/solr/collections/source_collection/leaders/shard1/leader
   [junit4]   2> 	2	/solr/collections/source_collection/leaders/shard2/leader
   [junit4]   2> 	2	/solr/collections/tmp_collection/leaders/shard2/leader
   [junit4]   2> 	2	/solr/collections/tmp_collection/leaders/shard1/leader
   [junit4]   2> 
   [junit4]   2> Maximum concurrent data watches above limit:
   [junit4]   2> 
   [junit4]   2> 	33	/solr/clusterstate.json
   [junit4]   2> 	8	/solr/collections/tmp_collection/cdcr/state/process
   [junit4]   2> 	8	/solr/collections/tmp_collection/cdcr/state/buffer
   [junit4]   2> 	7	/solr/collections/source_collection/cdcr/state/process
   [junit4]   2> 	6	/solr/collections/target_collection/cdcr/state/buffer
   [junit4]   2> 	6	/solr/collections/target_collection/cdcr/state/process
   [junit4]   2> 	4	/solr/collections/source_collection/state.json
   [junit4]   2> 	4	/solr/collections/source_collection/cdcr/state/buffer
   [junit4]   2> 	4	/solr/collections/target_collection/state.json
   [junit4]   2> 
   [junit4]   2> Maximum concurrent children watches above limit:
   [junit4]   2> 
   [junit4]   2> 	285	/solr/overseer/collection-queue-work
   [junit4]   2> 	67	/solr/overseer/queue
   [junit4]   2> 	33	/solr/collections
   [junit4]   2> 	32	/solr/live_nodes
   [junit4]   2> 	17	/solr/overseer/queue-work
   [junit4]   2> 
   [junit4]   2> 7903659 INFO  (SUITE-CdcrReplicationDistributedZkTest-seed#[DF3C5BBDBF828ECA]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.CdcrReplicationDistributedZkTest_DF3C5BBDBF828ECA-001
   [junit4]   2> Oct 01, 2016 5:13:04 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62), sim=RandomSimilarity(queryNorm=false): {}, locale=ar-MA, timezone=US/Central
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=189445672,total=500170752
   [junit4]   2> NOTE: All tests run in this JVM: [DistributedSpellCheckComponentTest, CursorPagingTest, TestSlowCompositeReaderWrapper, TestRTimerTree, TestSchemaResource, TestSimpleQParserPlugin, OverseerStatusTest, TestConfigSetsAPI, TestRecoveryHdfs, DistributedQueryElevationComponentTest, DateRangeFieldTest, FastVectorHighlighterTest, PolyFieldTest, TestLeaderElectionZkExpiry, RankQueryTest, SearchHandlerTest, OverseerCollectionConfigSetProcessorTest, DateMathParserTest, TestFieldCacheSanityChecker, TestJmxMonitoredMap, CleanupOldIndexTest, SpatialRPTFieldTypeTest, PingRequestHandlerTest, ZkNodePropsTest, ResponseHeaderTest, TestSolrCoreProperties, TestFaceting, HttpPartitionTest, QueryEqualityTest, ConnectionManagerTest, PluginInfoTest, BlockJoinFacetRandomTest, TestLFUCache, AnalysisAfterCoreReloadTest, LeaderInitiatedRecoveryOnCommitTest, TestUniqueKeyFieldResource, TestConfigReload, TestSolrCoreSnapshots, TestRemoteStreaming, TestSSLRandomization, TestRestManager, MultiThreadedOCPTest, TestFastWriter, TestFieldTypeCollectionResource, TestRandomDVFaceting, ConnectionReuseTest, BufferStoreTest, TestRandomFaceting, TestNumericTerms32, MultiTermTest, TestAuthenticationFramework, TestDFRSimilarityFactory, TestAnalyzedSuggestions, TestBackupRepositoryFactory, TestBlendedInfixSuggestions, HLLUtilTest, TestHdfsUpdateLog, TestSolrFieldCacheMBean, CollectionStateFormat2Test, DocExpirationUpdateProcessorFactoryTest, ExitableDirectoryReaderTest, HdfsNNFailoverTest, DistributedDebugComponentTest, TestFunctionQuery, TestJmxIntegration, TestCrossCoreJoin, RecoveryAfterSoftCommitTest, SSLMigrationTest, TestOmitPositions, TestMacroExpander, SignatureUpdateProcessorFactoryTest, TestJsonFacets, RequiredFieldsTest, CdcrBootstrapTest, AsyncCallRequestStatusResponseTest, TestExclusionRuleCollectionAccess, TestRawTransformer, TestSortByMinMaxFunction, PathHierarchyTokenizerFactoryTest, CacheHeaderTest, TestPKIAuthenticationPlugin, CurrencyFieldOpenExchangeTest, TestFieldCacheVsDocValues, TestSolrConfigHandler, LeaderElectionIntegrationTest, TestDynamicFieldResource, TestTrieFacet, TestStressCloudBlindAtomicUpdates, TestConfigSetProperties, TestImplicitCoreProperties, RulesTest, TestRandomFlRTGCloud, TestTolerantUpdateProcessorRandomCloud, TestGraphTermsQParserPlugin, NotRequiredUniqueKeyTest, ActionThrottleTest, TestLegacyNumericRangeQueryBuilder, TestBadConfig, SolrCLIZkUtilsTest, SmileWriterTest, SyncSliceTest, BasicZkTest, RecoveryZkTest, FullSolrCloudDistribCmdsTest, TestReplicationHandler, TestSolr4Spatial, TestGroupingSearch, HighlighterTest, SolrRequestParserTest, TestFoldingMultitermQuery, TestCSVLoader, SolrCoreCheckLockOnStartupTest, TestUpdate, FieldMutatingUpdateProcessorTest, DefaultValueUpdateProcessorTest, DocValuesMultiTest, DistanceFunctionTest, TestSolrDeletionPolicy1, XsltUpdateRequestHandlerTest, IndexBasedSpellCheckerTest, DisMaxRequestHandlerTest, DocumentAnalysisRequestHandlerTest, FileBasedSpellCheckerTest, TermVectorComponentTest, FieldAnalysisRequestHandlerTest, LoggingHandlerTest, ReturnFieldsTest, TestCSVResponseWriter, TestPartialUpdateDeduplication, TestQuerySenderListener, TestQuerySenderNoQuery, ResponseLogComponentTest, TestStressRecovery, TestHighlightDedupGrouping, AssignTest, CdcrReplicationDistributedZkTest]
   [junit4] Completed [635/635 (8!)] on J0 in 3096.27s, 11 tests, 3 errors <<< FAILURES!

[...truncated 11041 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:715: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid12170.hprof

Total time: 483 minutes 24 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1122 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1122/

9 tests failed.
FAILED:  org.apache.lucene.search.TestBooleanRewrites.testRandom

Error Message:
expected:<22> but was:<11>

Stack Trace:
java.lang.AssertionError: expected:<22> but was:<11>
	at __randomizedtesting.SeedInfo.seed([133FC438853637FC:6173E1373456818F]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.lucene.search.TestBooleanRewrites.assertEquals(TestBooleanRewrites.java:417)
	at org.apache.lucene.search.TestBooleanRewrites.testRandom(TestBooleanRewrites.java:363)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrRequestHandlerTest.testCheckpointActions

Error Message:
KeeperErrorCode = Session expired for /

Stack Trace:
org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:356)
	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:353)
	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:353)
	at org.apache.solr.common.cloud.SolrZkClient.printLayout(SolrZkClient.java:588)
	at org.apache.solr.common.cloud.SolrZkClient.printLayoutToStdOut(SolrZkClient.java:629)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForCollectionToDisappear(AbstractDistribZkTestBase.java:211)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForCollectionToDisappear(BaseCdcrDistributedZkTest.java:498)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.startServers(BaseCdcrDistributedZkTest.java:598)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.createSourceCollection(BaseCdcrDistributedZkTest.java:348)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.baseBefore(BaseCdcrDistributedZkTest.java:167)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:905)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsBasicDistributedZk2Test.test

Error Message:
Multiple exceptions

Stack Trace:
MultiException[java.lang.OutOfMemoryError: unable to create new native thread, java.util.concurrent.RejectedExecutionException: org.eclipse.jetty.io.ManagedSelector@7af7c615 id=0 keys=0 selected=0]
	at __randomizedtesting.SeedInfo.seed([D198B8B72273947A:59CC876D8C8FF982]:0)
	at org.eclipse.jetty.server.Server.doStart(Server.java:347)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:342)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner.start(JettySolrRunner.java:315)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJetty(AbstractFullDistribZkTestBase.java:510)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createJettys(AbstractFullDistribZkTestBase.java:399)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createServers(AbstractFullDistribZkTestBase.java:332)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:983)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
	Suppressed: java.util.concurrent.RejectedExecutionException: org.eclipse.jetty.io.ManagedSelector@7af7c615 id=0 keys=0 selected=0
		at org.eclipse.jetty.util.thread.QueuedThreadPool.execute(QueuedThreadPool.java:377)
		at org.eclipse.jetty.io.SelectorManager.execute(SelectorManager.java:125)
		at org.eclipse.jetty.io.SelectorManager.doStart(SelectorManager.java:255)
		at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
		at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:132)
		at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:106)
		at org.eclipse.jetty.server.AbstractConnector.doStart(AbstractConnector.java:260)
		at org.eclipse.jetty.server.AbstractNetworkConnector.doStart(AbstractNetworkConnector.java:81)
		at org.eclipse.jetty.server.ServerConnector.doStart(ServerConnector.java:244)
		at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
		at org.eclipse.jetty.server.Server.doStart(Server.java:384)
		... 39 more
Caused by: java.lang.OutOfMemoryError: unable to create new native thread
	at java.lang.Thread.start0(Native Method)
	at java.lang.Thread.start(Thread.java:714)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.startThreads(QueuedThreadPool.java:462)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.doStart(QueuedThreadPool.java:113)
	at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:132)
	at org.eclipse.jetty.server.Server.start(Server.java:405)
	at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:106)
	at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:61)
	at org.eclipse.jetty.server.Server.doStart(Server.java:372)
	... 39 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsBasicDistributedZk2Test

Error Message:
34 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsBasicDistributedZk2Test:     1) Thread[id=2887, name=watches-439-thread-2, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=2903, name=qtp1367269790-2903, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=2916, name=zkCallback-438-thread-4, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=2913, name=OverseerStateUpdate-96675738331054089-127.0.0.1:36896_-n_0000000001, state=TIMED_WAITING, group=Overseer state updater.]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)         at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:138)         at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:182)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=2864, name=qtp1754747651-2864, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=2861, name=qtp1754747651-2861, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=2882, name=zkCallback-438-thread-3, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=2878, name=TEST-HdfsBasicDistributedZk2Test.test-seed#[D198B8B72273947A]-EventThread, state=WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)    9) Thread[id=2880, name=zkCallback-438-thread-2, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=2865, name=qtp1754747651-2865, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   11) Thread[id=2886, name=watches-439-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   12) Thread[id=2866, name=qtp1754747651-2866, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   13) Thread[id=2905, name=qtp1367269790-2905, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   14) Thread[id=2914, name=OverseerCollectionConfigSetProcessor-96675738331054089-127.0.0.1:36896_-n_0000000001, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)         at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)         at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:336)         at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:299)         at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:279)         at org.apache.solr.cloud.DistributedQueue.peekElements(DistributedQueue.java:328)         at org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235)         at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:227)         at java.lang.Thread.run(Thread.java:745)   15) Thread[id=2904, name=qtp1367269790-2904, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   16) Thread[id=2906, name=qtp1367269790-2906, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   17) Thread[id=2902, name=qtp1367269790-2902, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   18) Thread[id=2915, name=OverseerHdfsCoreFailoverThread-96675738331054089-127.0.0.1:36896_-n_0000000001, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)         at java.lang.Thread.run(Thread.java:745)   19) Thread[id=2885, name=coreContainerWorkExecutor-897-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   20) Thread[id=2873, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   21) Thread[id=2877, name=TEST-HdfsBasicDistributedZk2Test.test-seed#[D198B8B72273947A]-SendThread(127.0.0.1:36395), state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)   22) Thread[id=2872, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   23) Thread[id=2899, name=coreZkRegister-896-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   24) Thread[id=2879, name=zkCallback-438-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   25) Thread[id=2862, name=qtp1754747651-2862-acceptor-0@ffe8692-ServerConnector@5884ee91{HTTP/1.1,[http/1.1]}{127.0.0.1:36896}, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)         at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)         at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:382)         at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:593)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)   26) Thread[id=2860, name=qtp1754747651-2860, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)   27) Thread[id=2895, name=searcherExecutor-902-thread-1, state=WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   28) Thread[id=2863, name=qtp1754747651-2863, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   29) Thread[id=2884, name=Thread-1243, state=WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1296)   30) Thread[id=2901, name=qtp1367269790-2901, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   31) Thread[id=2867, name=qtp1754747651-2867, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)         at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)         at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)         at java.lang.Thread.run(Thread.java:745)   32) Thread[id=2900, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   33) Thread[id=2917, name=zkCallback-438-thread-5, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   34) Thread[id=2868, name=org.eclipse.jetty.server.session.HashSessionManager@66792d02Timer, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 34 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsBasicDistributedZk2Test: 
   1) Thread[id=2887, name=watches-439-thread-2, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=2903, name=qtp1367269790-2903, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=2916, name=zkCallback-438-thread-4, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=2913, name=OverseerStateUpdate-96675738331054089-127.0.0.1:36896_-n_0000000001, state=TIMED_WAITING, group=Overseer state updater.]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:164)
        at org.apache.solr.cloud.DistributedQueue.peek(DistributedQueue.java:138)
        at org.apache.solr.cloud.Overseer$ClusterStateUpdater.run(Overseer.java:182)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=2864, name=qtp1754747651-2864, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=2861, name=qtp1754747651-2861, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=2882, name=zkCallback-438-thread-3, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=2878, name=TEST-HdfsBasicDistributedZk2Test.test-seed#[D198B8B72273947A]-EventThread, state=WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:494)
   9) Thread[id=2880, name=zkCallback-438-thread-2, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=2865, name=qtp1754747651-2865, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  11) Thread[id=2886, name=watches-439-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  12) Thread[id=2866, name=qtp1754747651-2866, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  13) Thread[id=2905, name=qtp1367269790-2905, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  14) Thread[id=2914, name=OverseerCollectionConfigSetProcessor-96675738331054089-127.0.0.1:36896_-n_0000000001, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryDelay(ZkCmdExecutor.java:102)
        at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:70)
        at org.apache.solr.common.cloud.SolrZkClient.getChildren(SolrZkClient.java:336)
        at org.apache.solr.cloud.DistributedQueue.fetchZkChildren(DistributedQueue.java:299)
        at org.apache.solr.cloud.DistributedQueue.firstChild(DistributedQueue.java:279)
        at org.apache.solr.cloud.DistributedQueue.peekElements(DistributedQueue.java:328)
        at org.apache.solr.cloud.OverseerTaskQueue.peekTopN(OverseerTaskQueue.java:235)
        at org.apache.solr.cloud.OverseerTaskProcessor.run(OverseerTaskProcessor.java:227)
        at java.lang.Thread.run(Thread.java:745)
  15) Thread[id=2904, name=qtp1367269790-2904, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  16) Thread[id=2906, name=qtp1367269790-2906, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  17) Thread[id=2902, name=qtp1367269790-2902, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  18) Thread[id=2915, name=OverseerHdfsCoreFailoverThread-96675738331054089-127.0.0.1:36896_-n_0000000001, state=TIMED_WAITING, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:137)
        at java.lang.Thread.run(Thread.java:745)
  19) Thread[id=2885, name=coreContainerWorkExecutor-897-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  20) Thread[id=2873, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  21) Thread[id=2877, name=TEST-HdfsBasicDistributedZk2Test.test-seed#[D198B8B72273947A]-SendThread(127.0.0.1:36395), state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:994)
  22) Thread[id=2872, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  23) Thread[id=2899, name=coreZkRegister-896-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  24) Thread[id=2879, name=zkCallback-438-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  25) Thread[id=2862, name=qtp1754747651-2862-acceptor-0@ffe8692-ServerConnector@5884ee91{HTTP/1.1,[http/1.1]}{127.0.0.1:36896}, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.nio.ch.ServerSocketChannelImpl.accept0(Native Method)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:422)
        at sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:250)
        at org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:382)
        at org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:593)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
  26) Thread[id=2860, name=qtp1754747651-2860, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
  27) Thread[id=2895, name=searcherExecutor-902-thread-1, state=WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  28) Thread[id=2863, name=qtp1754747651-2863, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  29) Thread[id=2884, name=Thread-1243, state=WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:502)
        at org.apache.solr.core.CloserThread.run(CoreContainer.java:1296)
  30) Thread[id=2901, name=qtp1367269790-2901, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  31) Thread[id=2867, name=qtp1754747651-2867, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:392)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.idleJobPoll(QueuedThreadPool.java:546)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.access$800(QueuedThreadPool.java:47)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:609)
        at java.lang.Thread.run(Thread.java:745)
  32) Thread[id=2900, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  33) Thread[id=2917, name=zkCallback-438-thread-5, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  34) Thread[id=2868, name=org.eclipse.jetty.server.session.HashSessionManager@66792d02Timer, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)
        at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([D198B8B72273947A]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsBasicDistributedZk2Test

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=2887, name=watches-439-thread-2, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=2915, name=OverseerHdfsCoreFailoverThread-96675738331054089-127.0.0.1:36896_-n_0000000001, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]         at java.lang.Thread.currentThread(Native Method)         at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=2916, name=zkCallback-438-thread-4, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=2885, name=coreContainerWorkExecutor-897-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=2877, name=TEST-HdfsBasicDistributedZk2Test.test-seed#[D198B8B72273947A]-SendThread(127.0.0.1:36395), state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at java.lang.Thread.sleep(Native Method)         at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)         at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)         at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)    6) Thread[id=2861, name=qtp1754747651-2861, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)         at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)         at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)         at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)         at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=2882, name=zkCallback-438-thread-3, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=2880, name=zkCallback-438-thread-2, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=2899, name=coreZkRegister-896-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=2879, name=zkCallback-438-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   11) Thread[id=2886, name=watches-439-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   12) Thread[id=2860, name=qtp1754747651-2860, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZk2Test]         at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)         at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)         at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)         at java.lang.Thread.run(Thread.java:745)   13) Thread[id=2895, name=searcherExecutor-902-thread-1, state=WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   14) Thread[id=2884, name=Thread-1243, state=WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:502)         at org.apache.solr.core.CloserThread.run(CoreContainer.java:1296)   15) Thread[id=2917, name=zkCallback-438-thread-5, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)         at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)         at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)   16) Thread[id=2868, name=org.eclipse.jetty.server.session.HashSessionManager@66792d02Timer, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093)         at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=2887, name=watches-439-thread-2, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=2915, name=OverseerHdfsCoreFailoverThread-96675738331054089-127.0.0.1:36896_-n_0000000001, state=RUNNABLE, group=Overseer Hdfs SolrCore Failover Thread.]
        at java.lang.Thread.currentThread(Native Method)
        at org.apache.solr.cloud.OverseerAutoReplicaFailoverThread.run(OverseerAutoReplicaFailoverThread.java:139)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=2916, name=zkCallback-438-thread-4, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=2885, name=coreContainerWorkExecutor-897-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=2877, name=TEST-HdfsBasicDistributedZk2Test.test-seed#[D198B8B72273947A]-SendThread(127.0.0.1:36395), state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.zookeeper.client.StaticHostProvider.next(StaticHostProvider.java:101)
        at org.apache.zookeeper.ClientCnxn$SendThread.startConnect(ClientCnxn.java:940)
        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1003)
   6) Thread[id=2861, name=qtp1754747651-2861, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method)
        at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269)
        at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93)
        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:101)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:233)
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=2882, name=zkCallback-438-thread-3, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=2880, name=zkCallback-438-thread-2, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=2899, name=coreZkRegister-896-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=2879, name=zkCallback-438-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  11) Thread[id=2886, name=watches-439-thread-1, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215)
        at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460)
        at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
        at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1066)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
  12) Thread[id=2860, name=qtp1754747651-2860, state=RUNNABLE, group=TGRP-HdfsBasicDistributedZk2Test]
        at org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:181)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:171)
        at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.execute(ExecuteProduceConsume.java:102)
        at org.eclipse.jetty.io.ManagedSelector.run(ManagedSelector.java:137)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
        at java.lang.Thread.run(Thread.java:745)
  13) Thread[id=2895, name=searcherExecutor-902-thread-1, state=WAITING, group=TGRP-HdfsBasicDistributedZk2Test]
        at sun.misc.Unsafe

[...truncated too long message...]

3) Uninverting(_3(7.0.0):c1) Uninverting(_4(7.0.0):c1)))}
   [junit4]   2> 2504820 INFO  (SUITE-TestSortingResponseWriter-seed#[D198B8B72273947A]-worker) [    ] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 2504820 INFO  (SUITE-TestSortingResponseWriter-seed#[D198B8B72273947A]-worker) [    ] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{commit=} 0 57
   [junit4]   2> 2504843 INFO  (TEST-TestSortingResponseWriter.testSortingOutput-seed#[D198B8B72273947A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testSortingOutput
   [junit4]   2> 2504851 INFO  (TEST-TestSortingResponseWriter.testSortingOutput-seed#[D198B8B72273947A]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:1&qt=/export&fl=floatdv,intdv,stringdv,longdv,doubledv&sort=intdv+asc&wt=xml} hits=1 status=0 QTime=7
   [junit4]   2> 2504897 INFO  (TEST-TestSortingResponseWriter.testSortingOutput-seed#[D198B8B72273947A]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:7&qt=/export&fl=floatdv,intdv,stringdv,longdv,doubledv&sort=intdv+asc&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 2504937 INFO  (TEST-TestSortingResponseWriter.testSortingOutput-seed#[D198B8B72273947A]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:1&qt=/export&fl=intdv_m,floatdv_m,doubledv_m,longdv_m,stringdv_m&sort=intdv+asc&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 2504957 INFO  (TEST-TestSortingResponseWriter.testSortingOutput-seed#[D198B8B72273947A]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:7&qt=/export&fl=intdv_m,floatdv_m,doubledv_m,longdv_m,stringdv_m&sort=intdv+asc&wt=xml} hits=1 status=0 QTime=0
   [junit4]   2> 2504984 INFO  (TEST-TestSortingResponseWriter.testSortingOutput-seed#[D198B8B72273947A]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:(1+2)&qt=/export&fl=intdv&sort=intdv+desc&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 2505170 INFO  (TEST-TestSortingResponseWriter.testSortingOutput-seed#[D198B8B72273947A]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:(1+2)&qt=/export&fl=intdv&sort=intdv+asc&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 2505177 INFO  (TEST-TestSortingResponseWriter.testSortingOutput-seed#[D198B8B72273947A]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:(1+7)&qt=/export&fl=intdv&sort=stringdv+desc&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 2505274 INFO  (TEST-TestSortingResponseWriter.testSortingOutput-seed#[D198B8B72273947A]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:(1+7)&qt=/export&fl=intdv&sort=stringdv+asc&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 2505349 INFO  (TEST-TestSortingResponseWriter.testSortingOutput-seed#[D198B8B72273947A]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:(1+2)&qt=/export&fl=intdv&sort=floatdv+asc,intdv+desc&wt=xml} hits=2 status=0 QTime=0
   [junit4]   2> 2505392 INFO  (TEST-TestSortingResponseWriter.testSortingOutput-seed#[D198B8B72273947A]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:(1+2)&qt=/export&fl=intdv&sort=floatdv+desc,intdv+asc&wt=xml} hits=2 status=0 QTime=2
   [junit4]   2> 2505456 INFO  (TEST-TestSortingResponseWriter.testSortingOutput-seed#[D198B8B72273947A]) [    ] o.a.s.c.S.Request [collection1]  webapp=null path=null params={q=id:(1+2+3)&qt=/export&fl=intdv&sort=floatdv+asc,stringdv+asc,intdv+desc&wt=xml} hits=3 status=0 QTime=0
   [junit4]   2> 2505473 INFO  (TEST-TestSortingResponseWriter.testSortingOutput-seed#[D198B8B72273947A]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testSortingOutput
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSortingResponseWriter -Dtests.method=testSortingOutput -Dtests.seed=D198B8B72273947A -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=lt -Dtests.timezone=Africa/Lubumbashi -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] FAILURE 0.65s J0 | TestSortingResponseWriter.testSortingOutput <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: docs were sent out-of-order: lastDocID=2 vs doc=0
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D198B8B72273947A:65682FD62F3C9EDA]:0)
   [junit4]    > 	at org.apache.solr.response.SortingResponseWriter$FloatValue.setCurrentValue(SortingResponseWriter.java:1061)
   [junit4]    > 	at org.apache.solr.response.SortingResponseWriter$TripleValueSortDoc.setValues(SortingResponseWriter.java:679)
   [junit4]    > 	at org.apache.solr.response.SortingResponseWriter.write(SortingResponseWriter.java:179)
   [junit4]    > 	at org.apache.solr.util.TestHarness.query(TestHarness.java:316)
   [junit4]    > 	at org.apache.solr.util.TestHarness.query(TestHarness.java:292)
   [junit4]    > 	at org.apache.solr.response.TestSortingResponseWriter.testSortingOutput(TestSortingResponseWriter.java:151)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 2505474 INFO  (SUITE-TestSortingResponseWriter-seed#[D198B8B72273947A]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> 2505474 INFO  (SUITE-TestSortingResponseWriter-seed#[D198B8B72273947A]-worker) [    ] o.a.s.c.CoreContainer Shutting down CoreContainer instance=41599093
   [junit4]   2> 2505499 INFO  (coreCloseExecutor-4340-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@2d5bdde4
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.response.TestSortingResponseWriter_D198B8B72273947A-001
   [junit4]   2> NOTE: test params are: codec=Lucene62, sim=ClassicSimilarity, locale=lt, timezone=Africa/Lubumbashi
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=220016888,total=511705088
   [junit4]   2> NOTE: All tests run in this JVM: [GraphQueryTest, RuleEngineTest, DefaultValueUpdateProcessorTest, TestJmxMonitoredMap, TestJoin, TestLazyCores, RegexBoostProcessorTest, AnalyticsMergeStrategyTest, DateMathParserTest, TestBadConfig, DirectoryFactoryTest, CdcrRequestHandlerTest, StandardRequestHandlerTest, TestAuthorizationFramework, TestFoldingMultitermQuery, HdfsRecoveryZkTest, DistributedSpellCheckComponentTest, TestCodecSupport, PeerSyncTest, MinimalSchemaTest, DistributedQueryComponentCustomSortTest, SolrCLIZkUtilsTest, TestTolerantUpdateProcessorCloud, TestOmitPositions, FastVectorHighlighterTest, DistributedFacetExistsSmallTest, TestCryptoKeys, TestPivotHelperCode, TestUninvertingReader, ZkStateReaderTest, TestPseudoReturnFields, TestLeaderInitiatedRecoveryThread, CheckHdfsIndexTest, TestTestInjection, TestSha256AuthenticationProvider, TestComplexPhraseQParserPlugin, TestCSVLoader, ZkNodePropsTest, TestLFUCache, OverseerStatusTest, TestFieldCacheReopen, TestAddFieldRealTimeGet, TestManagedSynonymFilterFactory, DeleteReplicaTest, CoreAdminCreateDiscoverTest, PathHierarchyTokenizerFactoryTest, TestClassicSimilarityFactory, SuggestComponentTest, SystemInfoHandlerTest, TestInitQParser, ReplicationFactorTest, TolerantUpdateProcessorTest, TestFunctionQuery, ReturnFieldsTest, TestRequestStatusCollectionAPI, TestAnalyzedSuggestions, DisMaxRequestHandlerTest, UUIDFieldTest, TestSolrCoreProperties, TestCloudPivotFacet, NumericFieldsTest, TestRebalanceLeaders, BasicZkTest, XsltUpdateRequestHandlerTest, DistribCursorPagingTest, BitVectorTest, TestShardHandlerFactory, TestExclusionRuleCollectionAccess, SyncSliceTest, FacetPivotSmallTest, ConnectionManagerTest, TestFieldCacheSortRandom, MergeStrategyTest, TestCollationFieldDocValues, TestAuthenticationFramework, TestRangeQuery, QueryEqualityTest, TestReplicationHandlerBackup, LeaderElectionIntegrationTest, InfoHandlerTest, SolrRequestParserTest, SharedFSAutoReplicaFailoverTest, TestIntervalFaceting, CdcrBootstrapTest, DeleteInactiveReplicaTest, QueryElevationComponentTest, CloneFieldUpdateProcessorFactoryTest, TestConfigSetsAPI, ChangedSchemaMergeTest, TestDFRSimilarityFactory, OverseerCollectionConfigSetProcessorTest, TestSuggestSpellingConverter, TestReRankQParserPlugin, DistributedQueueTest, HdfsLockFactoryTest, CleanupOldIndexTest, AnalysisAfterCoreReloadTest, LeaderInitiatedRecoveryOnCommitTest, TestOrdValues, TestSortingResponseWriter]
   [junit4] Completed [633/633 (4!)] on J0 in 1.29s, 1 test, 1 failure <<< FAILURES!

[...truncated 1 lines...]
   [junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20160929_130427_689.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid12261.hprof ...
   [junit4] Heap dump file created [533377174 bytes in 9.381 secs]
   [junit4] <<< JVM J0: EOF ----

[...truncated 11038 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:715: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid12261.hprof

Total time: 324 minutes 52 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1121 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1121/

No tests ran.

Build Log:
[...truncated 1895 lines...]
   [junit4] Suite: org.apache.lucene.search.TestFuzzyQuery
   [junit4]   2> 9월 25, 2016 4:08:54 오전 com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.search.TestFuzzyQuery
   [junit4]   2>    1) Thread[id=2972, name=SUITE-TestFuzzyQuery-seed#[AEF39117F58BD628], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1556)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:688)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:685)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:685)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:701)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:681)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:64)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:414)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:681)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:140)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:591)
   [junit4]   2>    2) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1249)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1323)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:601)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:450)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:243)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:354)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:10)
   [junit4]   2>    3) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:47)
   [junit4]   2>    4) Thread[id=2973, name=TEST-TestFuzzyQuery.testRandom-seed#[AEF39117F58BD628], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:254)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:151)
   [junit4]   2>         at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:730)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.randomSimpleString(TestFuzzyQuery.java:506)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.testRandom(TestFuzzyQuery.java:517)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestFuzzyQuery -Dtests.method=testRandom -Dtests.seed=AEF39117F58BD628 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=ko-KR -Dtests.timezone=Africa/Mogadishu -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   7200s J2 | TestFuzzyQuery.testRandom <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AEF39117F58BD628]:0)
   [junit4]   2> 9월 25, 2016 4:08:54 오전 com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> 9월 25, 2016 4:09:14 오전 com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.search.TestFuzzyQuery: 
   [junit4]   2>    1) Thread[id=2973, name=TEST-TestFuzzyQuery.testRandom-seed#[AEF39117F58BD628], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:151)
   [junit4]   2>         at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:730)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.randomSimpleString(TestFuzzyQuery.java:506)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.testRandom(TestFuzzyQuery.java:517)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 9월 25, 2016 4:09:14 오전 com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=2973, name=TEST-TestFuzzyQuery.testRandom-seed#[AEF39117F58BD628], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2> 9월 25, 2016 4:09:17 오전 com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=2973, name=TEST-TestFuzzyQuery.testRandom-seed#[AEF39117F58BD628], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.getRandomness(RandomizedContext.java:110)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.getRandom(RandomizedContext.java:132)
   [junit4]   2>         at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:730)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.randomSimpleString(TestFuzzyQuery.java:506)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.testRandom(TestFuzzyQuery.java:517)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {field=PostingsFormat(name=Memory doPackFST= false)}, docValues:{}, maxPointsInLeafNode=1489, maxMBSortInHeap=7.472815105601821, sim=RandomSimilarity(queryNorm=true): {field=DFR I(F)B3(800.0)}, locale=ko-KR, timezone=Africa/Mogadishu
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=2,free=276095768,total=426770432
   [junit4]   2> NOTE: All tests run in this JVM: [TestParallelReaderEmptyIndex, TestSingleInstanceLockFactory, TestBlockPostingsFormat, TestStressIndexing, TestAtomicUpdate, TestIndexWriterDeleteByQuery, TestBytesRefArray, TestToken, TestCodecHoldsOpenFiles, Test2BBinaryDocValues, TestCachingCollector, TestDocumentWriter, TestIndexWriterMergePolicy, TestArrayUtil, TestNot, TestNotDocIdSet, TestRegexpRandom2, TestIndexInput, TestInPlaceMergeSorter, TestTermsEnum, TestFilterWeight, TestSpanFirstQuery, TestSameScoresWithThreads, TestCollectionUtil, TestConjunctionDISI, TestIndexWriterExceptions, TestSparseFixedBitSet, TestMatchNoDocsQuery, TestUpgradeIndexMergePolicy, TestBinaryTerms, TestSimpleSearchEquivalence, TestReaderWrapperDVTypeCheck, TestNoMergeScheduler, TestSegmentTermDocs, TestIndexWriterReader, TestNRTCachingDirectory, TestSegmentMerger, TestTopDocsMerge, TestTerms, TestRegexpRandom, TestIntroSorter, TestDocIdsWriter, TestTermQuery, TestFixedBitDocIdSet, TestDirectMonotonic, TestTransactionRollback, TestIntsRef, TestTransactions, TestVirtualMethod, TestIndexWriterThreadsToSegments, TestHighCompressionMode, TestLogMergePolicy, TestSpanCollection, TestBlendedTermQuery, TestPointQueries, TestGeoUtils, LimitedFiniteStringsIteratorTest, TestParallelTermEnum, TestSnapshotDeletionPolicy, TestMergeSchedulerExternal, TestIndexWriterMaxDocs, TestOrdinalMap, TestWordlistLoader, TestDocInverterPerFieldErrorInfo, TestSpans, TestRateLimiter, TestTwoPhaseCommitTool, TestDocValuesScoring, TestLazyProxSkipping, TestMultiTermsEnum, TestWildcard, TestIndexWriterMerging, TestFSTs, TestMultiMMap, TestDeletionPolicy, TestFlushByRamOrCountsPolicy, TestIndexWriterDelete, TestNRTThreads, TestMultiThreadTermVectors, TestSort, TestMultiDocValues, TestSpanExplanations, TestStressIndexing2, TestCodecs, TestParallelLeafReader, TestSloppyPhraseQuery2, TestFuzzyQuery]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestFuzzyQuery -Dtests.seed=AEF39117F58BD628 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=ko-KR -Dtests.timezone=Africa/Mogadishu -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J2 | TestFuzzyQuery (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AEF39117F58BD628]:0)
   [junit4] Completed [433/433 (1!)] on J2 in 7223.45s, 11 tests, 2 errors <<< FAILURES!

[...truncated 9384 lines...]
   [junit4] Suite: org.apache.solr.update.SoftAutoCommitTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.update.SoftAutoCommitTest_BF4DC0948A35D7ED-001/init-core-data-001
   [junit4]   2> 13712 INFO  (SUITE-SoftAutoCommitTest-seed#[BF4DC0948A35D7ED]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 13713 INFO  (SUITE-SoftAutoCommitTest-seed#[BF4DC0948A35D7ED]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore
   [junit4]   2> 13714 INFO  (SUITE-SoftAutoCommitTest-seed#[BF4DC0948A35D7ED]-worker) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 14233 WARN  (SUITE-SoftAutoCommitTest-seed#[BF4DC0948A35D7ED]-worker) [    ] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 14256 INFO  (SUITE-SoftAutoCommitTest-seed#[BF4DC0948A35D7ED]-worker) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 14628 INFO  (SUITE-SoftAutoCommitTest-seed#[BF4DC0948A35D7ED]-worker) [    ] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 14842 INFO  (SUITE-SoftAutoCommitTest-seed#[BF4DC0948A35D7ED]-worker) [    ] o.a.s.s.IndexSchema [null] Schema name=test
   [junit4]   2> 16984 INFO  (SUITE-SoftAutoCommitTest-seed#[BF4DC0948A35D7ED]-worker) [    ] o.a.s.s.IndexSchema [null] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 16986 INFO  (SUITE-SoftAutoCommitTest-seed#[BF4DC0948A35D7ED]-worker) [    ] o.a.s.s.IndexSchema [null] unique key field: id
   [junit4]   2> 17407 INFO  (coreLoadExecutor-18-thread-1) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 17567 WARN  (coreLoadExecutor-18-thread-1) [    ] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 17569 INFO  (coreLoadExecutor-18-thread-1) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 17685 INFO  (coreLoadExecutor-18-thread-1) [    ] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 18076 INFO  (coreLoadExecutor-18-thread-1) [    ] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 18406 INFO  (coreLoadExecutor-18-thread-1) [    ] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 18407 INFO  (coreLoadExecutor-18-thread-1) [    ] o.a.s.s.IndexSchema [collection1] unique key field: id
   [junit4]   2> 18540 INFO  (coreLoadExecutor-18-thread-1) [    ] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1
   [junit4]   2> 18541 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.update.SoftAutoCommitTest_BF4DC0948A35D7ED-001/init-core-data-001/]
   [junit4]   2> 18561 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.SolrCore New index directory detected: old=null new=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.update.SoftAutoCommitTest_BF4DC0948A35D7ED-001/init-core-data-001/index/
   [junit4]   2> 18562 WARN  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1] Solr index directory '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.update.SoftAutoCommitTest_BF4DC0948A35D7ED-001/init-core-data-001/index' doesn't exist. Creating new index...
   [junit4]   2> 18562 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=29, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 18581 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@1ccd07a0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@417b113),segFN=segments_1,generation=1}
   [junit4]   2> 18581 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 18589 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 1 plugins of type queryResponseWriter: [xml]
   [junit4]   2> 18618 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 1 plugins of type queryParser: [foo]
   [junit4]   2> 18755 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 4 plugins of type searchComponent: [spellcheck, termsComp, tvComponent, highlight]
   [junit4]   2> 18755 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 18786 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 18802 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 18802 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 18802 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 18803 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 18803 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 18807 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 18807 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 18851 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 18857 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 19028 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 39 plugins of type requestHandler: [/update, /update/json, /update/csv, /update/json/docs, /config, /schema, /replication, /get, /admin/ping, /admin/segments, /admin/luke, /admin/system, /admin/mbeans, /admin/plugins, /admin/threads, /admin/properties, /admin/logging, /admin/file, /export, /graph, /stream, /sql, /terms, standard, dismax, mock, defaults, lazy, spellCheckCompRH, spellCheckCompRH_Direct, spellCheckWithWordbreak, spellCheckWithWordbreak_Direct, spellCheckCompRH1, mltrh, tvrh, /mlt, /debug/dump, /search-facet-def, /search-facet-invariants]
   [junit4]   2> 19028 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,standard,spellCheckCompRH,lazy,/update/json/docs,/admin/luke,/export,spellCheckWithWordbreak_Direct,tvrh,/get,/admin/properties,/update/json,mltrh,/admin/threads,/search-facet-def,/update/csv,/sql,/graph,/search-facet-invariants,/admin/segments,spellCheckCompRH1,/mlt,/admin/system,/replication,dismax,defaults,/config,/stream,/schema,/admin/plugins,/admin/logging,spellCheckWithWordbreak,mock,/admin/ping,/update,/admin/file,/terms,spellCheckCompRH_Direct,/debug/dump
   [junit4]   2> 19046 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 19046 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 19067 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 19068 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 19068 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=45, maxMergeAtOnceExplicit=45, maxMergedSegmentMB=97.3837890625, floorSegmentMB=1.2412109375, forceMergeDeletesPctAllowed=25.59226891409414, segmentsPerTier=33.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8687551454356546
   [junit4]   2> 19070 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@1ccd07a0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@417b113),segFN=segments_1,generation=1}
   [junit4]   2> 19071 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 19072 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@46d7e4b[collection1] main]
   [junit4]   2> 19081 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.r.RestManager Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 19082 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 19083 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 19083 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 19246 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 19468 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 19470 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 19474 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 19474 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 19474 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 19475 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 19475 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 19476 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 19476 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 19477 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 19477 INFO  (searcherExecutor-19-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@46d7e4b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 19480 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.UpdateLog Looking up max value of version field to seed version buckets
   [junit4]   2> 19480 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.VersionInfo Refreshing highest value of _version_ for 65536 version buckets from index
   [junit4]   2> 19480 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.VersionInfo No terms found for _version_, cannot seed version bucket highest value from index
   [junit4]   2> 19480 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1546413749740503040
   [junit4]   2> 19506 INFO  (coreLoadExecutor-18-thread-1) [    x:collection1] o.a.s.u.UpdateLog Took 25.0ms to seed version buckets with highest version 1546413749740503040
   [junit4]   2> 19509 INFO  (SUITE-SoftAutoCommitTest-seed#[BF4DC0948A35D7ED]-worker) [    ] o.a.s.SolrTestCaseJ4 ####initCore end
   [junit4]   2> 19512 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testSoftAndHardCommitMaxTimeDelete
   [junit4]   2> 19513 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    ] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 20046 WARN  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    ] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 20055 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 20123 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    ] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 20213 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    ] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 21003 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    ] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 21004 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    ] o.a.s.s.IndexSchema [collection1] unique key field: id
   [junit4]   2> 21118 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    ] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1
   [junit4]   2> 21119 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.update.SoftAutoCommitTest_BF4DC0948A35D7ED-001/init-core-data-001/]
   [junit4]   2> 21120 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrCore New index directory detected: old=null new=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.update.SoftAutoCommitTest_BF4DC0948A35D7ED-001/init-core-data-001/index/
   [junit4]   2> 21121 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 1 plugins of type queryResponseWriter: [xml]
   [junit4]   2> 21124 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 1 plugins of type queryParser: [foo]
   [junit4]   2> 21132 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 4 plugins of type searchComponent: [spellcheck, termsComp, tvComponent, highlight]
   [junit4]   2> 21133 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 21149 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 21149 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 21149 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 21150 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 21163 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 21163 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 21184 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 21184 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 21184 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 21185 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 21217 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 39 plugins of type requestHandler: [/update, /update/json, /update/csv, /update/json/docs, /config, /schema, /replication, /get, /admin/ping, /admin/segments, /admin/luke, /admin/system, /admin/mbeans, /admin/plugins, /admin/threads, /admin/properties, /admin/logging, /admin/file, /export, /graph, /stream, /sql, /terms, standard, dismax, mock, defaults, lazy, spellCheckCompRH, spellCheckCompRH_Direct, spellCheckWithWordbreak, spellCheckWithWordbreak_Direct, spellCheckCompRH1, mltrh, tvrh, /mlt, /debug/dump, /search-facet-def, /search-facet-invariants]
   [junit4]   2> 21218 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,standard,spellCheckCompRH,lazy,/update/json/docs,/admin/luke,/export,spellCheckWithWordbreak_Direct,tvrh,/get,/admin/properties,/update/json,mltrh,/admin/threads,/search-facet-def,/update/csv,/sql,/graph,/search-facet-invariants,/admin/segments,spellCheckCompRH1,/mlt,/admin/system,/replication,dismax,defaults,/config,/stream,/schema,/admin/plugins,/admin/logging,spellCheckWithWordbreak,mock,/admin/ping,/update,/admin/file,/terms,spellCheckCompRH_Direct,/debug/dump
   [junit4]   2> 21224 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 21224 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 21232 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4e9d835[collection1] main]
   [junit4]   2> 21233 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.r.RestManager Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 21233 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 21233 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 21234 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 21240 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 21261 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 21261 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 21262 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 21262 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 21262 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 21262 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 21263 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 21263 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 21263 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 21264 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 21264 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@4e9d835[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 21264 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.UpdateLog Looking up max value of version field to seed version buckets
   [junit4]   2> 21265 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.VersionInfo Refreshing highest value of _version_ for 65536 version buckets from index
   [junit4]   2> 21265 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.VersionInfo No terms found for _version_, cannot seed version bucket highest value from index
   [junit4]   2> 21265 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1546413751612211200
   [junit4]   2> 21268 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.UpdateLog Took 3.0ms to seed version buckets with highest version 1546413751612211200
   [junit4]   2> 21271 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.DefaultSolrCoreState Closing old IndexWriter... core=collection1
   [junit4]   2> 21288 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@2e8d29de
   [junit4]   2> 21289 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@1ccd07a0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@417b113),segFN=segments_1,generation=1}
   [junit4]   2> 21289 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 21289 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 21290 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@4b49546e[collection1] main]
   [junit4]   2> 21290 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@4b49546e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 21291 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@5c4c6b29
   [junit4]   2> 21291 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.DirectUpdateHandler2 closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=0,transaction_logs_total_number=0}
   [junit4]   2> 21291 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrCore [collection1] Closing main searcher on request.
   [junit4]   2> 21560 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[529 (1546413751642619904)]} 0 267
   [junit4]   2> 21615 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 21615 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@3b468ce5
   [junit4]   2> 21744 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@1ccd07a0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@417b113),segFN=segments_1,generation=1}
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@1ccd07a0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@417b113),segFN=segments_2,generation=2}
   [junit4]   2> 21744 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 2
   [junit4]   2> 21751 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@6474949a[collection1] main]
   [junit4]   2> 21751 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 21751 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@6474949a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(7.0.0):c1)))}
   [junit4]   2> 21776 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{commit=} 0 161
   [junit4]   2> 21785 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{delete=[529 (-1546413752149082112)]} 0 7
   [junit4]   2> 22285 INFO  (commitScheduler-27-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 22332 INFO  (commitScheduler-27-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@5e7985b7[collection1] main]
   [junit4]   2> 22333 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@5e7985b7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 22338 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[550 (1546413752734187520)]} 0 3
   [junit4]   2> 22345 INFO  (commitScheduler-27-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 22844 INFO  (commitScheduler-27-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 22996 INFO  (commitScheduler-26-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 23036 INFO  (commitScheduler-27-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@72c4be05[collection1] main]
   [junit4]   2> 23038 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@72c4be05[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(7.0.0):c1)))}
   [junit4]   2> 23038 INFO  (commitScheduler-26-thread-1) [    x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@3b468ce5
   [junit4]   2> 23039 INFO  (commitScheduler-26-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@1ccd07a0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@417b113),segFN=segments_2,generation=2}
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@1ccd07a0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@417b113),segFN=segments_3,generation=3}
   [junit4]   2> 23039 INFO  (commitScheduler-26-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 3
   [junit4]   2> 23040 INFO  (commitScheduler-26-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@183a7b14[collection1] main]
   [junit4]   2> 23040 INFO  (searcherExecutor-24-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@183a7b14[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(7.0.0):c1)))}
   [junit4]   2> 23041 INFO  (commitScheduler-26-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 23044 INFO  (commitScheduler-27-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 25040 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.SolrTestCaseJ4 ###Ending testSoftAndHardCommitMaxTimeDelete
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SoftAutoCommitTest -Dtests.method=testSoftAndHardCommitMaxTimeDelete -Dtests.seed=BF4DC0948A35D7ED -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=ar-IQ -Dtests.timezone=Etc/GMT-8 -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 5.55s J1 | SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: Got a soft commit we weren't expecting
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BF4DC0948A35D7ED:78017809919D1A5D]:0)
   [junit4]    > 	at org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeDelete(SoftAutoCommitTest.java:287)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 25068 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.SolrTestCaseJ4 ###Starting testSoftAndHardCommitMaxTimeMixedAdds
   [junit4]   2> 25081 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 25160 WARN  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 25162 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 25184 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 25199 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 25600 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 25625 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.IndexSchema [collection1] unique key field: id
   [junit4]   2> 25734 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1
   [junit4]   2> 25735 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.update.SoftAutoCommitTest_BF4DC0948A35D7ED-001/init-core-data-001/]
   [junit4]   2> 25752 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrCore New index directory detected: old=null new=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.update.SoftAutoCommitTest_BF4DC0948A35D7ED-001/init-core-data-001/index/
   [junit4]   2> 25760 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 1 plugins of type queryResponseWriter: [xml]
   [junit4]   2> 25762 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 1 plugins of type queryParser: [foo]
   [junit4]   2> 25763 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 4 plugins of type searchComponent: [spellcheck, termsComp, tvComponent, highlight]
   [junit4]   2> 25764 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 25764 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 25764 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 25764 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 25764 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 25765 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 25765 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 25766 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 25766 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 25766 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 25766 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 25803 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 39 plugins of type requestHandler: [/update, /update/json, /update/csv, /update/json/docs, /config, /schema, /replication, /get, /admin/ping, /admin/segments, /admin/luke, /admin/system, /admin/mbeans, /admin/plugins, /admin/threads, /admin/properties, /admin/logging, /admin/file, /export, /graph, /stream, /sql, /terms, standard, dismax, mock, defaults, lazy, spellCheckCompRH, spellCheckCompRH_Direct, spellCheckWithWordbreak, spellCheckWithWordbreak_Direct, spellCheckCompRH1, mltrh, tvrh, /mlt, /debug/dump, /search-facet-def, /search-facet-invariants]
   [junit4]   2> 25803 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,standard,spellCheckCompRH,lazy,/update/json/docs,/admin/luke,/export,spellCheckWithWordbreak_Direct,tvrh,/get,/admin/properties,/update/json,mltrh,/admin/threads,/search-facet-def,/update/csv,/sql,/graph,/search-facet-invariants,/admin/segments,spellCheckCompRH1,/mlt,/admin/system,/replication,dismax,defaults,/config,/stream,/schema,/admin/plugins,/admin/logging,spellCheckWithWordbreak,mock,/admin/ping,/update,/admin/file,/terms,spellCheckCompRH_Direct,/debug/dump
   [junit4]   2> 25910 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 25910 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 25910 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@3419a255[collection1] main]
   [junit4]   2> 25911 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.r.RestManager Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 25911 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 25911 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 25911 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 25928 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 25957 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 25958 INFO  (searcherExecutor-28-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 25959 INFO  (searcherExecutor-28-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 25959 INFO  (searcherExecutor-28-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [junit4]   2> 25959 INFO  (searcherExecutor-28-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: multipleFields
   [junit4]   2> 25959 INFO  (searcherExecutor-28-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 25960 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.UpdateLog Looking up max value of version field to seed version buckets
   [junit4]   2> 25964 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.VersionInfo Refreshing highest value of _version_ for 65536 version buckets from index
   [junit4]   2> 25964 INFO  (searcherExecutor-28-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: external
   [junit4]   2> 25964 INFO  (searcherExecutor-28-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: freq
   [junit4]   2> 25965 INFO  (searcherExecutor-28-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: fqcn
   [junit4]   2> 25965 INFO  (searcherExecutor-28-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: perDict
   [junit4]   2> 25965 INFO  (searcherExecutor-28-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@3419a255[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(7.0.0):c1)))}
   [junit4]   2> 25979 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.VersionInfo Found MAX value 1546413752734187520 from Terms for _version_ in index
   [junit4]   2> 25983 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.UpdateLog Took 23.0ms to seed version buckets with highest version 1546413752734187520
   [junit4]   2> 25998 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.DefaultSolrCoreState Closing old IndexWriter... core=collection1
   [junit4]   2> 26000 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=20, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4716157541174105]
   [junit4]   2> 26007 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@1ccd07a0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@417b113),segFN=segments_3,generation=3}
   [junit4]   2> 26007 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 3
   [junit4]   2> 26007 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.DefaultSolrCoreState New IndexWriter is ready to be used.
   [junit4]   2> 26012 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@23be85e3[collection1] main]
   [junit4]   2> 26013 INFO  (searcherExecutor-28-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@23be85e3[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(7.0.0):c1)))}
   [junit4]   2> 26013 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrCore [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@e0c4225
   [junit4]   2> 26017 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.DirectUpdateHandler2 closing DirectUpdateHandler2{commits=4,autocommit maxTime=1200ms,autocommits=1,soft autocommit maxTime=500ms,soft autocommits=2,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=2,cumulative_deletesById=1,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=294,transaction_logs_total_number=2}
   [junit4]   2> 26024 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrCore [collection1] Closing main searcher on request.
   [junit4]   2> 26044 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[529 (1546413756604481536)]} 0 18
   [junit4]   2> 26732 INFO  (commitScheduler-31-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 26761 INFO  (commitScheduler-31-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@69703024[collection1] main]
   [junit4]   2> 26762 INFO  (searcherExecutor-28-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@69703024[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(7.0.0):c1) Uninverting(_2(7.0.0):c1)))}
   [junit4]   2> 26763 INFO  (commitScheduler-31-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 26774 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.LogUpdateProcessorFactory [collection1]  webapp=null path=null params={}{add=[530 (1546413757386719232)]} 0 2
   [junit4]   2> 27256 INFO  (commitScheduler-30-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 27256 INFO  (commitScheduler-30-thread-1) [    x:collection1] o.a.s.u.SolrIndexWriter Calling setCommitData with IW:org.apache.solr.update.SolrIndexWriter@334ba346
   [junit4]   2> 27294 INFO  (commitScheduler-30-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@1ccd07a0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@417b113),segFN=segments_3,generation=3}
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@1ccd07a0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@417b113),segFN=segments_4,generation=4}
   [junit4]   2> 27294 INFO  (commitScheduler-30-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 4
   [junit4]   2> 27297 INFO  (commitScheduler-30-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@601550bb[collection1] main]
   [junit4]   2> 27297 INFO  (commitScheduler-30-thread-1) [    x:collection1] o.a.s.u.DirectUpdateHandler2 end_commit_flush
   [junit4]   2> 27300 INFO  (searcherExecutor-28-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@601550bb[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_1(7.0.0):c1) Uninverting(_2(7.0.0):c1) Uninverting(_3(7.0.0):c1)))}
   [junit4]   2> 30566 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.SolrTestCaseJ4 ###Ending testSoftAndHardCommitMaxTimeMixedAdds
   [junit4]   2> 30569 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.SolrTestCaseJ4 ###Starting testSoftAndHardCommitMaxTimeRapidAdds
   [junit4]   2> 30570 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrResourceLoader [null] Added 2 libs to classloader, from paths: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/lib/classes]
   [junit4]   2> 30978 WARN  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 30985 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 31209 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 31308 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 32195 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.IndexSchema [collection1] default search field in schema is text. WARNING: Deprecated, please use 'df' on request instead.
   [junit4]   2> 32228 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.IndexSchema [collection1] unique key field: id
   [junit4]   2> 32379 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.CoreContainer Reloading SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1
   [junit4]   2> 32408 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.update.SoftAutoCommitTest_BF4DC0948A35D7ED-001/init-core-data-001/]
   [junit4]   2> 32409 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.SolrCore New index directory detected: old=null new=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.update.SoftAutoCommitTest_BF4DC0948A35D7ED-001/init-core-data-001/index/
   [junit4]   2> 32464 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 1 plugins of type queryResponseWriter: [xml]
   [junit4]   2> 32466 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 1 plugins of type queryParser: [foo]
   [junit4]   2> 32467 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 4 plugins of type searchComponent: [spellcheck, termsComp, tvComponent, highlight]
   [junit4]   2> 32467 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 32467 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 32467 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 32467 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 32467 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 32467 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 32467 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 32468 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 32468 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 32478 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 32479 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.p.UpdateRequestProcessorChain inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 32676 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.PluginBag [collection1] Initialized 39 plugins of type requestHandler: [/update, /update/json, /update/csv, /update/json/docs, /config, /schema, /replication, /get, /admin/ping, /admin/segments, /admin/luke, /admin/system, /admin/mbeans, /admin/plugins, /admin/threads, /admin/properties, /admin/logging, /admin/file, /export, /graph, /stream, /sql, /terms, standard, dismax, mock, defaults, lazy, spellCheckCompRH, spellCheckCompRH_Direct, spellCheckWithWordbreak, spellCheckWithWordbreak_Direct, spellCheckCompRH1, mltrh, tvrh, /mlt, /debug/dump, /search-facet-def, /search-facet-invariants]
   [junit4]   2> 32677 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,standard,spellCheckCompRH,lazy,/update/json/docs,/admin/luke,/export,spellCheckWithWordbreak_Direct,tvrh,/get,/admin/properties,/update/json,mltrh,/admin/threads,/search-facet-def,/update/csv,/sql,/graph,/search-facet-invariants,/admin/segments,spellCheckCompRH1,/mlt,/admin/system,/replication,dismax,defaults,/config,/stream,/schema,/admin/plugins,/admin/logging,spellCheckWithWordbreak,mock,/admin/ping,/update,/admin/file,/terms,spellCheckCompRH_Direct,/debug/dump
   [junit4]   2> 32680 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 32680 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 32693 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@2d6d6afc[collection1] main]
   [junit4]   2> 32694 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.r.RestManager Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 32694 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 32694 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 32694 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.h.c.SpellCheckComponent Initializing spell checkers
   [junit4]   2> 32698 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.s.DirectSolrSpellChecker init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 32734 INFO  (TEST-SoftAutoCommitTest.testSoftAndHardCommitMaxTimeRapidAdds-seed#[BF4DC0948A35D7ED]) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 32734 INFO  (searcherExecutor-32-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: default
   [junit4]   2> 32736 INFO  (searcherExecutor-32-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: direct
   [junit4]   2> 32736 INFO  (searcherExecutor-32-thread-1-processing-x:collection1) [    x:collection1] o.a.s.h.c.SpellCheckComponent Loading spell index for spellchecker: wordbreak
   [ju

[...truncated too long message...]

til.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2>    2) Thread[id=256394, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrVersionReplicationTest]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Sep 25, 2016 5:27:13 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=256390, name=searcherExecutor-5664-thread-1, state=WAITING, group=TGRP-CdcrVersionReplicationTest]
   [junit4]   2>    2) Thread[id=256394, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrVersionReplicationTest]
   [junit4]   2> Sep 25, 2016 5:27:16 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=256390, name=searcherExecutor-5664-thread-1, state=WAITING, group=TGRP-CdcrVersionReplicationTest]
   [junit4]   2>         at sun.misc.Unsafe.park(Native Method)
   [junit4]   2>         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]   2>         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]   2>         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]   2>         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {}, docValues:{}, maxPointsInLeafNode=780, maxMBSortInHeap=5.38963644527261, sim=ClassicSimilarity, locale=es-GT, timezone=Atlantic/Cape_Verde
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=2,free=71710832,total=498073600
   [junit4]   2> NOTE: All tests run in this JVM: [DisMaxRequestHandlerTest, SharedFSAutoReplicaFailoverTest, SolrRequestParserTest, TestQuerySenderNoQuery, TestSolrDeletionPolicy2, TestJmxIntegration, TestSubQueryTransformer, DistributedQueryElevationComponentTest, TestCloudSchemaless, TestSolrQueryResponse, TestFieldSortValues, TestRandomDVFaceting, TestFastWriter, TestJoin, ActionThrottleTest, TestExactStatsCache, ClusterStateTest, TestSolrCoreProperties, TestAtomicUpdateErrorCases, AnalysisErrorHandlingTest, TestExclusionRuleCollectionAccess, TestReloadDeadlock, TestSolr4Spatial, SolrCLIZkUtilsTest, TestOverriddenPrefixQueryForCustomFieldType, CdcrBootstrapTest, SuggestComponentTest, DistribCursorPagingTest, CollectionsAPISolrJTest, DistributedQueueTest, TestConfigSetImmutable, TestFieldTypeResource, ZkStateReaderTest, TestLeaderInitiatedRecoveryThread, TestFaceting, MBeansHandlerTest, TestUseDocValuesAsStored2, TestPerFieldSimilarity, TestSQLHandler, TestBadConfig, DocumentAnalysisRequestHandlerTest, CleanupOldIndexTest, TestNumericTerms64, VersionInfoTest, TestMaxScoreQueryParser, TestNonDefinedSimilarityFactory, DeleteReplicaTest, StatsReloadRaceTest, TestBulkSchemaConcurrent, BasicDistributedZkTest, TestDistributedSearch, ClassificationUpdateProcessorFactoryTest, DistributedSpellCheckComponentTest, TestFieldCollectionResource, ResourceLoaderTest, SolrCoreTest, HighlighterConfigTest, HttpPartitionTest, TestCodecSupport, BasicZkTest, FastVectorHighlighterTest, TestComponentsName, LeaderInitiatedRecoveryOnCommitTest, TestFieldCacheWithThreads, RecoveryAfterSoftCommitTest, TestConfigSetsAPI, TestHashQParserPlugin, SSLMigrationTest, DateFieldTest, TestManagedStopFilterFactory, IgnoreCommitOptimizeUpdateProcessorFactoryTest, UUIDFieldTest, TestNamedUpdateProcessors, TestPivotHelperCode, BooleanFieldTest, TestCloudPivotFacet, TestCrossCoreJoin, TestRemoteStreaming, PolyFieldTest, OverseerCollectionConfigSetProcessorTest, TestCryptoKeys, CheckHdfsIndexTest, RegexBoostProcessorTest, LeaderElectionIntegrationTest, TestDocumentBuilder, DistributedQueryComponentCustomSortTest, TestFieldCacheSort, TestDefaultStatsCache, TestGraphMLResponseWriter, CursorMarkTest, TestTolerantUpdateProcessorCloud, CollectionStateFormat2Test, TestReplicationHandlerBackup, TestFieldCache, OverseerStatusTest, DistributedFacetPivotSmallAdvancedTest, TestBinaryResponseWriter, QueryEqualityTest, StandardRequestHandlerTest, TestSolrQueryParserDefaultOperatorResource, BigEndianAscendingWordSerializerTest, TestLazyCores, CursorPagingTest, CdcrVersionReplicationTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=CdcrVersionReplicationTest -Dtests.seed=BF4DC0948A35D7ED -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=es-GT -Dtests.timezone=Atlantic/Cape_Verde -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J0 | CdcrVersionReplicationTest (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.CdcrVersionReplicationTest: 
   [junit4]    >    1) Thread[id=256390, name=searcherExecutor-5664-thread-1, state=WAITING, group=TGRP-CdcrVersionReplicationTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    2) Thread[id=256394, name=Connection evictor, state=TIMED_WAITING, group=TGRP-CdcrVersionReplicationTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BF4DC0948A35D7ED]:0)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   [junit4]    >    1) Thread[id=256390, name=searcherExecutor-5664-thread-1, state=WAITING, group=TGRP-CdcrVersionReplicationTest]
   [junit4]    >         at sun.misc.Unsafe.park(Native Method)
   [junit4]    >         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
   [junit4]    >         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
   [junit4]    >         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
   [junit4]    >         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BF4DC0948A35D7ED]:0)
   [junit4] Completed [467/636 (7!)] on J0 in 787.70s, 1 test, 3 errors <<< FAILURES!

[...truncated 509 lines...]
   [junit4] JVM J1: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20160925_034407_301.sysout
   [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid25432.hprof ...
   [junit4] Heap dump file created [511477987 bytes in 10.210 secs]
   [junit4] <<< JVM J1: EOF ----

[...truncated 3535 lines...]
ERROR: Step ‘Archive the artifacts’ failed: no workspace for Lucene-Solr-NightlyTests-master #1121
ERROR: Step ‘Publish JUnit test result report’ failed: no workspace for Lucene-Solr-NightlyTests-master #1121
ERROR: lucene is offline; cannot locate JDK 1.8 (latest)
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any
ERROR: lucene is offline; cannot locate JDK 1.8 (latest)
ERROR: lucene is offline; cannot locate JDK 1.8 (latest)




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1120 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1120/

270 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.BasicFunctionalityTest

Error Message:
java.lang.NullPointerException

Stack Trace:
java.lang.RuntimeException: java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([219E416F25A4A93]:0)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:75)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:602)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:595)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:437)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:426)
	at org.apache.solr.BasicFunctionalityTest.beforeTests(BasicFunctionalityTest.java:66)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:811)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:178)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:142)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:171)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:73)
	... 29 more


FAILED:  junit.framework.TestSuite.org.apache.solr.ConvertedLegacyTest

Error Message:
java.lang.NullPointerException

Stack Trace:
java.lang.RuntimeException: java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([219E416F25A4A93]:0)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:75)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:602)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:595)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:437)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:426)
	at org.apache.solr.ConvertedLegacyTest.beforeTests(ConvertedLegacyTest.java:37)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:811)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:178)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:142)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:171)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:73)
	... 29 more


FAILED:  org.apache.solr.DistributedIntervalFacetingTest.test

Error Message:
Error from server at http://127.0.0.1:58331//collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.NullPointerException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: null,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: null  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1119)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:252)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:414)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:266)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:217)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(Thread.java:745) Caused by: java.lang.NullPointerException  at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)  at org.apache.solr.core.SolrResourceLoader.&lt;init&gt;(SolrResourceLoader.java:178)  at org.apache.solr.core.ConfigSetService$Default.createCoreResourceLoader(ConfigSetService.java:159)  at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:74)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:862)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:520)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:58331//collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.NullPointerException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: null,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: null
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1119)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:252)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:414)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:266)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:217)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)
	at org.apache.solr.core.SolrResourceLoader.&lt;init&gt;(SolrResourceLoader.java:178)
	at org.apache.solr.core.ConfigSetService$Default.createCoreResourceLoader(ConfigSetService.java:159)
	at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:74)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:862)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:520)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([219E416F25A4A93:8A4DDBCC5CA6276B]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.BaseDistributedSearchTestCase.del(BaseDistributedSearchTestCase.java:540)
	at org.apache.solr.DistributedIntervalFacetingTest.test(DistributedIntervalFacetingTest.java:44)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsRepeatStatement.callStatement(BaseDistributedSearchTestCase.java:1011)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.EchoParamsTest

Error Message:
java.lang.NullPointerException

Stack Trace:
java.lang.RuntimeException: java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([219E416F25A4A93]:0)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:75)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:602)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:595)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:437)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:426)
	at org.apache.solr.EchoParamsTest.beforeClass(EchoParamsTest.java:29)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:811)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:178)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:142)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:171)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:73)
	... 29 more


FAILED:  junit.framework.TestSuite.org.apache.solr.MinimalSchemaTest

Error Message:
java.lang.NullPointerException

Stack Trace:
java.lang.RuntimeException: java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([219E416F25A4A93]:0)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:75)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:602)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:595)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:437)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:426)
	at org.apache.solr.MinimalSchemaTest.beforeClass(MinimalSchemaTest.java:39)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:811)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:178)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:142)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:171)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:73)
	... 29 more


FAILED:  junit.framework.TestSuite.org.apache.solr.SampleTest

Error Message:
java.lang.NullPointerException

Stack Trace:
java.lang.RuntimeException: java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([219E416F25A4A93]:0)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:75)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:602)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:595)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:437)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:426)
	at org.apache.solr.SampleTest.beforeClass(SampleTest.java:41)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:811)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:178)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:142)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:171)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:73)
	... 29 more


FAILED:  junit.framework.TestSuite.org.apache.solr.SolrInfoMBeanTest

Error Message:
java.lang.NullPointerException

Stack Trace:
java.lang.RuntimeException: java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([219E416F25A4A93]:0)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:75)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:602)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:595)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:437)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:426)
	at org.apache.solr.SolrInfoMBeanTest.beforeClass(SolrInfoMBeanTest.java:40)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:811)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:178)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:142)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:171)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:73)
	... 29 more


FAILED:  junit.framework.TestSuite.org.apache.solr.SolrTestCaseJ4Test

Error Message:
Can't load schema /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.SolrTestCaseJ4Test_219E416F25A4A93-001/tempDir-001/core1/conf/schema-tiny.xml: null

Stack Trace:
org.apache.solr.common.SolrException: Can't load schema /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.SolrTestCaseJ4Test_219E416F25A4A93-001/tempDir-001/core1/conf/schema-tiny.xml: null
	at __randomizedtesting.SeedInfo.seed([219E416F25A4A93]:0)
	at org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:607)
	at org.apache.solr.schema.IndexSchema.<init>(IndexSchema.java:183)
	at org.apache.solr.schema.IndexSchemaFactory.create(IndexSchemaFactory.java:56)
	at org.apache.solr.schema.IndexSchemaFactory.buildIndexSchema(IndexSchemaFactory.java:75)
	at org.apache.solr.util.TestHarness.<init>(TestHarness.java:96)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:605)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:595)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:437)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:444)
	at org.apache.solr.SolrTestCaseJ4Test.beforeClass(SolrTestCaseJ4Test.java:52)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:811)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at org.apache.solr.schema.IndexSchema.readSchema(IndexSchema.java:583)
	... 33 more


FAILED:  org.apache.solr.TestCursorMarkWithoutUniqueKey.test

Error Message:
java.lang.NullPointerException

Stack Trace:
java.lang.RuntimeException: java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([219E416F25A4A93:8A4DDBCC5CA6276B]:0)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:75)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:602)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:595)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:437)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:426)
	at org.apache.solr.TestCursorMarkWithoutUniqueKey.beforeSetupCore(TestCursorMarkWithoutUniqueKey.java:38)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:905)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:178)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:142)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:171)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:73)
	... 43 more


FAILED:  org.apache.solr.TestDistributedGrouping.test

Error Message:
Error from server at https://127.0.0.1:55533//collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.NullPointerException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: null,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: null  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1119)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:252)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:414)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:266)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:217)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(Thread.java:745) Caused by: java.lang.NullPointerException  at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)  at org.apache.solr.core.SolrResourceLoader.&lt;init&gt;(SolrResourceLoader.java:178)  at org.apache.solr.core.ConfigSetService$Default.createCoreResourceLoader(ConfigSetService.java:159)  at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:74)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:862)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:520)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:55533//collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.NullPointerException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: null,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: null
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1119)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:252)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:414)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:266)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:217)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)
	at org.apache.solr.core.SolrResourceLoader.&lt;init&gt;(SolrResourceLoader.java:178)
	at org.apache.solr.core.ConfigSetService$Default.createCoreResourceLoader(ConfigSetService.java:159)
	at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:74)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:862)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:520)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([219E416F25A4A93:8A4DDBCC5CA6276B]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.BaseDistributedSearchTestCase.del(BaseDistributedSearchTestCase.java:540)
	at org.apache.solr.TestDistributedGrouping.test(TestDistributedGrouping.java:53)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsRepeatStatement.callStatement(BaseDistributedSearchTestCase.java:1011)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.TestDistributedMissingSort.test

Error Message:
Error from server at https://127.0.0.1:46248//collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.NullPointerException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: null,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: null  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1119)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:252)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:414)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:266)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:217)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(Thread.java:745) Caused by: java.lang.NullPointerException  at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)  at org.apache.solr.core.SolrResourceLoader.&lt;init&gt;(SolrResourceLoader.java:178)  at org.apache.solr.core.ConfigSetService$Default.createCoreResourceLoader(ConfigSetService.java:159)  at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:74)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:862)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:520)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:46248//collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.NullPointerException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: null,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: null
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1119)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:252)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:414)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:266)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:217)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)
	at org.apache.solr.core.SolrResourceLoader.&lt;init&gt;(SolrResourceLoader.java:178)
	at org.apache.solr.core.ConfigSetService$Default.createCoreResourceLoader(ConfigSetService.java:159)
	at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:74)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:862)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:520)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([219E416F25A4A93:8A4DDBCC5CA6276B]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.BaseDistributedSearchTestCase.del(BaseDistributedSearchTestCase.java:540)
	at org.apache.solr.TestDistributedMissingSort.index(TestDistributedMissingSort.java:48)
	at org.apache.solr.TestDistributedMissingSort.test(TestDistributedMissingSort.java:42)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsRepeatStatement.callStatement(BaseDistributedSearchTestCase.java:1011)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.TestDistributedSearch.test

Error Message:
Error from server at http://127.0.0.1:49317//collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.NullPointerException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: null,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: null  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1119)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:252)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:414)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:266)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:217)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(Thread.java:745) Caused by: java.lang.NullPointerException  at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)  at org.apache.solr.core.SolrResourceLoader.&lt;init&gt;(SolrResourceLoader.java:178)  at org.apache.solr.core.ConfigSetService$Default.createCoreResourceLoader(ConfigSetService.java:159)  at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:74)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:862)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:520)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:49317//collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.NullPointerException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: null,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: null
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1119)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:252)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:414)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:266)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:217)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)
	at org.apache.solr.core.SolrResourceLoader.&lt;init&gt;(SolrResourceLoader.java:178)
	at org.apache.solr.core.ConfigSetService$Default.createCoreResourceLoader(ConfigSetService.java:159)
	at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:74)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:862)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:520)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([219E416F25A4A93:8A4DDBCC5CA6276B]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.BaseDistributedSearchTestCase.del(BaseDistributedSearchTestCase.java:540)
	at org.apache.solr.TestDistributedSearch.test(TestDistributedSearch.java:109)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsRepeatStatement.callStatement(BaseDistributedSearchTestCase.java:1011)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.TestGroupingSearch

Error Message:
java.lang.NullPointerException

Stack Trace:
java.lang.RuntimeException: java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([219E416F25A4A93]:0)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:75)
	at org.apache.solr.SolrTestCaseJ4.createCore(SolrTestCaseJ4.java:602)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:595)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:437)
	at org.apache.solr.SolrTestCaseJ4.initCore(SolrTestCaseJ4.java:426)
	at org.apache.solr.TestGroupingSearch.beforeTests(TestGroupingSearch.java:70)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:811)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.NullPointerException
	at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:178)
	at org.apache.solr.core.SolrResourceLoader.<init>(SolrResourceLoader.java:142)
	at org.apache.solr.core.SolrConfig.<init>(SolrConfig.java:171)
	at org.apache.solr.util.TestHarness.createConfig(TestHarness.java:73)
	... 29 more


FAILED:  org.apache.solr.TestSimpleTrackingShardHandler.testSolrXmlOverrideAndCorrectShardHandler

Error Message:
Error from server at https://127.0.0.1:54893/_cso/x/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /_cso/x/collection1/select. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.NullPointerException},msg=SolrCore &apos;collection1&apos; is not available due to init failure: null,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: null  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1119)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:252)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:414)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:266)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:217)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(Thread.java:745) Caused by: java.lang.NullPointerException  at org.apache.solr.core.SolrResourceLoader.addToClassLoader(SolrResourceLoader.java:202)  at org.apache.solr.core.SolrResourceLoader.&lt;init&gt;(SolrResourceLoader.java:178)  at org.apache.solr.core.ConfigSetService$Default.createCoreResourceLoader(ConfigSetService.java:159)  at org.apache.solr.core.ConfigSetService.getConfig(ConfigSetService.java:74)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:862)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:520)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  ... 1 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:54893/_cso/x/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>

[...truncated too long message...]

b/apacheds-interceptors-event-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-exception-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-journal-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-normalization-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-operational-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-referral-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-schema-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-subtree-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-interceptors-trigger-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-jdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-jdbm1-2.0.0-M2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-kerberos-codec-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-ldif-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-mavibot-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-kerberos-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/bcprov-jdk15-1.45.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/cglib-nodep-2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/commons-math3-3.4.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/easymock-3.0.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-common-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-hdfs-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-minikdc-2.7.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-annotations-2.5.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-core-asl-1.9.13.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-databind-2.5.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-mapper-asl-1.9.13.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/mockito-core-1.9.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/objenesis-1.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/presto-parser-0.122.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/slice-0.10.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/analysis/icu/lucene-analyzers-icu-7.0.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/analysis-extras/lib/icu4j-56.1.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_102/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.3.4.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20160922_223528_905.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20160922_223528_905.suites -stdin
   [junit4] ERROR: JVM J2 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1501)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:121)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:955)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:952)
   [junit4] 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4] 	at java.lang.Thread.run(Thread.java:745)

BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:707: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build.xml:252: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/common-build.xml:536: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:1443: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:984: At least one slave process threw an exception, first: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.

Total time: 513 minutes 47 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1119 - Failure

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1119/

11 tests failed.
FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=55549, name=collection2, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=55549, name=collection2, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:59186/vlp
	at __randomizedtesting.SeedInfo.seed([8EB1B6904EB41305]:0)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:998)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:59186/vlp
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:415)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:367)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1289)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1059)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:1001)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1599)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1620)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:988)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 11 more


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
Timeout waiting for all live and active

Stack Trace:
java.lang.AssertionError: Timeout waiting for all live and active
	at __randomizedtesting.SeedInfo.seed([8EB1B6904EB41305:6E5894AE0487EFD]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:309)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:127)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.core.TestDynamicLoading.testDynamicLoading

Error Message:
Could not get expected value  'X val changed' for path 'x' full output: {   "responseHeader":{     "status":0,     "QTime":0},   "params":{"wt":"json"},   "context":{     "webapp":"",     "path":"/test1",     "httpMethod":"GET"},   "class":"org.apache.solr.core.BlobStoreTestRequestHandler",   "x":null},  from server:  null

Stack Trace:
java.lang.AssertionError: Could not get expected value  'X val changed' for path 'x' full output: {
  "responseHeader":{
    "status":0,
    "QTime":0},
  "params":{"wt":"json"},
  "context":{
    "webapp":"",
    "path":"/test1",
    "httpMethod":"GET"},
  "class":"org.apache.solr.core.BlobStoreTestRequestHandler",
  "x":null},  from server:  null
	at __randomizedtesting.SeedInfo.seed([8EB1B6904EB41305:56FC9BC7B969B6A5]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.solr.core.TestSolrConfigHandler.testForResponseElement(TestSolrConfigHandler.java:535)
	at org.apache.solr.core.TestDynamicLoading.testDynamicLoading(TestDynamicLoading.java:249)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1474364963730,generation=2,filelist=[_5.fdt, _5.fdx, _5.fnm, _5.nvd, _5.nvm, _5.si, _5_Lucene50_0.doc, _5_Lucene50_0.tim, _5_Lucene50_0.tip, _7.fdt, _7.fdx, _7.fnm, _7.nvd, _7.nvm, _7.si, _7_Lucene50_0.doc, _7_Lucene50_0.tim, _7_Lucene50_0.tip, _8.cfe, _8.cfs, _8.si, segments_2]}]> but was:<[{indexVersion=1474364963730,generation=2,filelist=[_5.fdt, _5.fdx, _5.fnm, _5.nvd, _5.nvm, _5.si, _5_Lucene50_0.doc, _5_Lucene50_0.tim, _5_Lucene50_0.tip, _7.fdt, _7.fdx, _7.fnm, _7.nvd, _7.nvm, _7.si, _7_Lucene50_0.doc, _7_Lucene50_0.tim, _7_Lucene50_0.tip, _8.cfe, _8.cfs, _8.si, segments_2]}, {indexVersion=1474364963730,generation=3,filelist=[_a.fdt, _a.fdx, _a.fnm, _a.nvd, _a.nvm, _a.si, _a_Lucene50_0.doc, _a_Lucene50_0.tim, _a_Lucene50_0.tip, segments_3]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1474364963730,generation=2,filelist=[_5.fdt, _5.fdx, _5.fnm, _5.nvd, _5.nvm, _5.si, _5_Lucene50_0.doc, _5_Lucene50_0.tim, _5_Lucene50_0.tip, _7.fdt, _7.fdx, _7.fnm, _7.nvd, _7.nvm, _7.si, _7_Lucene50_0.doc, _7_Lucene50_0.tim, _7_Lucene50_0.tip, _8.cfe, _8.cfs, _8.si, segments_2]}]> but was:<[{indexVersion=1474364963730,generation=2,filelist=[_5.fdt, _5.fdx, _5.fnm, _5.nvd, _5.nvm, _5.si, _5_Lucene50_0.doc, _5_Lucene50_0.tim, _5_Lucene50_0.tip, _7.fdt, _7.fdx, _7.fnm, _7.nvd, _7.nvm, _7.si, _7_Lucene50_0.doc, _7_Lucene50_0.tim, _7_Lucene50_0.tip, _8.cfe, _8.cfs, _8.si, segments_2]}, {indexVersion=1474364963730,generation=3,filelist=[_a.fdt, _a.fdx, _a.fnm, _a.nvd, _a.nvm, _a.si, _a_Lucene50_0.doc, _a_Lucene50_0.tim, _a_Lucene50_0.tip, segments_3]}]>
	at __randomizedtesting.SeedInfo.seed([8EB1B6904EB41305:AB66ADA03EFC1D06]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1227)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.handler.component.DistributedFacetPivotLongTailTest.test

Error Message:
Error from server at https://127.0.0.1:49359//collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    java.lang.OutOfMemoryError: GC overhead limit exceeded</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:49359//collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    java.lang.OutOfMemoryError: GC overhead limit exceeded</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([8EB1B6904EB41305:6E5894AE0487EFD]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:152)
	at org.apache.solr.handler.component.DistributedFacetPivotLongTailTest.test(DistributedFacetPivotLongTailTest.java:71)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.lucene.search.TestFuzzyQuery.testRandom

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([75FB2BA2C4CAA695]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.search.TestFuzzyQuery

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([75FB2BA2C4CAA695]:0)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart

Error Message:
Timeout while trying to assert number of documents @ source_collection

Stack Trace:
java.lang.AssertionError: Timeout while trying to assert number of documents @ source_collection
	at __randomizedtesting.SeedInfo.seed([8EB1B6904EB41305:D2AC51118094A733]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:273)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart(CdcrReplicationDistributedZkTest.java:257)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.AssertionError: expected:<110> but was:<106>
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:264)
	... 42 more


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterLeaderChange

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard1

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard1
	at __randomizedtesting.SeedInfo.seed([8EB1B6904EB41305:5C41FA73101BB537]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterLeaderChange(CdcrReplicationDistributedZkTest.java:305)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
	at __randomizedtesting.SeedInfo.seed([8EB1B6904EB41305]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.GeneratedMethodAccessor84.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest:     1) Thread[id=119548, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=119553, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=119543, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=119573, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=119558, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=119538, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=119563, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=119578, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=119568, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=119533, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 
   1) Thread[id=119548, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=119553, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=119543, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=119573, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=119558, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=119538, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=119563, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=119578, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=119568, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=119533, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([8EB1B6904EB41305]:0)




Build Log:
[...truncated 1631 lines...]
   [junit4] Suite: org.apache.lucene.search.TestFuzzyQuery
   [junit4]   2> Rgs 20, 2016 3:56:40 AM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.search.TestFuzzyQuery
   [junit4]   2>    1) Thread[id=47, name=SUITE-TestFuzzyQuery-seed#[75FB2BA2C4CAA695], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1556)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:688)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:685)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:685)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:701)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:681)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:64)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:414)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:681)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:140)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:591)
   [junit4]   2>    2) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1249)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1323)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:601)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:450)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:243)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:354)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:10)
   [junit4]   2>    3) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:47)
   [junit4]   2>    4) Thread[id=48, name=TEST-TestFuzzyQuery.testRandom-seed#[75FB2BA2C4CAA695], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2>         at java.lang.Object.hashCode(Native Method)
   [junit4]   2>         at java.util.WeakHashMap.hash(WeakHashMap.java:298)
   [junit4]   2>         at java.util.WeakHashMap.getEntry(WeakHashMap.java:426)
   [junit4]   2>         at java.util.WeakHashMap.containsKey(WeakHashMap.java:417)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.context(RandomizedContext.java:254)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.current(RandomizedContext.java:151)
   [junit4]   2>         at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:729)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.randomSimpleString(TestFuzzyQuery.java:506)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.testRandom(TestFuzzyQuery.java:517)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestFuzzyQuery -Dtests.method=testRandom -Dtests.seed=75FB2BA2C4CAA695 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=lt-LT -Dtests.timezone=America/Argentina/Rio_Gallegos -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   7199s J1 | TestFuzzyQuery.testRandom <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([75FB2BA2C4CAA695]:0)
   [junit4]   2> Rgs 20, 2016 3:56:40 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> Rgs 20, 2016 3:57:00 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 1 thread leaked from SUITE scope at org.apache.lucene.search.TestFuzzyQuery: 
   [junit4]   2>    1) Thread[id=48, name=TEST-TestFuzzyQuery.testRandom-seed#[75FB2BA2C4CAA695], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2>         at java.util.WeakHashMap.getTable(WeakHashMap.java:350)
   [junit4]   2>         at java.util.WeakHashMap.get(WeakHashMap.java:397)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.getPerThread(RandomizedContext.java:315)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.getRandomness(RandomizedContext.java:110)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.getRandom(RandomizedContext.java:132)
   [junit4]   2>         at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:729)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.randomSimpleString(TestFuzzyQuery.java:506)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.testRandom(TestFuzzyQuery.java:517)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Rgs 20, 2016 3:57:00 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=48, name=TEST-TestFuzzyQuery.testRandom-seed#[75FB2BA2C4CAA695], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2> Rgs 20, 2016 3:57:03 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=48, name=TEST-TestFuzzyQuery.testRandom-seed#[75FB2BA2C4CAA695], state=RUNNABLE, group=TGRP-TestFuzzyQuery]
   [junit4]   2>         at java.lang.Object.hashCode(Native Method)
   [junit4]   2>         at java.util.WeakHashMap.hash(WeakHashMap.java:298)
   [junit4]   2>         at java.util.WeakHashMap.get(WeakHashMap.java:396)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.getPerThread(RandomizedContext.java:315)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.getRandomness(RandomizedContext.java:110)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedContext.getRandom(RandomizedContext.java:132)
   [junit4]   2>         at org.apache.lucene.util.LuceneTestCase.random(LuceneTestCase.java:729)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.randomSimpleString(TestFuzzyQuery.java:506)
   [junit4]   2>         at org.apache.lucene.search.TestFuzzyQuery.testRandom(TestFuzzyQuery.java:517)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {field=FSTOrd50}, docValues:{}, maxPointsInLeafNode=365, maxMBSortInHeap=5.939115795628311, sim=ClassicSimilarity, locale=lt-LT, timezone=America/Argentina/Rio_Gallegos
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=2,free=204973928,total=244318208
   [junit4]   2> NOTE: All tests run in this JVM: [TestRecyclingIntBlockAllocator, TestSumDocFreq, TestFuzzyQuery]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestFuzzyQuery -Dtests.seed=75FB2BA2C4CAA695 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=lt-LT -Dtests.timezone=America/Argentina/Rio_Gallegos -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J1 | TestFuzzyQuery (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([75FB2BA2C4CAA695]:0)
   [junit4] Completed [364/433 (1!)] on J1 in 7223.48s, 3 tests, 2 errors <<< FAILURES!

[...truncated 10031 lines...]
   [junit4] Suite: org.apache.solr.handler.TestReplicationHandler
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/init-core-data-001
   [junit4]   2> 1484458 INFO  (SUITE-TestReplicationHandler-seed#[8EB1B6904EB41305]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
   [junit4]   2> 1484459 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testEmptyCommits
   [junit4]   2> 1484460 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/collection1
   [junit4]   2> 1484475 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 1484554 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@a4a36aa{/solr,null,AVAILABLE}
   [junit4]   2> 1484555 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.e.j.s.ServerConnector Started ServerConnector@7e702bb7{HTTP/1.1,[http/1.1]}{127.0.0.1:40836}
   [junit4]   2> 1484555 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.e.j.s.Server Started @1492810ms
   [junit4]   2> 1484555 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/collection1/data, hostContext=/solr, hostPort=40836}
   [junit4]   2> 1484555 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 1484555 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001'
   [junit4]   2> 1484555 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1484555 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 1484556 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/solr.xml
   [junit4]   2> 1484560 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/.
   [junit4]   2> 1484561 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.CoreContainer New CoreContainer 1219314704
   [junit4]   2> 1484561 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001]
   [junit4]   2> 1484561 WARN  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.CoreContainer Couldn't add files from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/lib to classpath: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/lib
   [junit4]   2> 1484561 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,
   [junit4]   2> 1484567 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.u.UpdateShardHandler Created UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&maxConnections=100000&maxConnectionsPerHost=100000
   [junit4]   2> 1484567 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1484567 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1484567 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 1484567 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 1484568 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/.
   [junit4]   2> 1484569 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection1, config=solrconfig.xml, loadOnStartup=true, schema=schema.xml, configSetProperties=configsetprops.json, transient=false, dataDir=data/}
   [junit4]   2> 1484569 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.CorePropertiesLocator Found core collection1 in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/./collection1
   [junit4]   2> 1484569 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions
   [junit4]   2> 1484569 INFO  (coreLoadExecutor-2198-thread-1) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/collection1'
   [junit4]   2> 1484586 INFO  (coreLoadExecutor-2198-thread-1) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1484586 INFO  (coreLoadExecutor-2198-thread-1) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 1484590 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.s.SolrDispatchFilter user.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0
   [junit4]   2> 1484590 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 1484599 INFO  (coreLoadExecutor-2198-thread-1) [    ] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 1484926 WARN  (coreLoadExecutor-2198-thread-1) [    ] o.a.s.c.Config Beginning with Solr 5.5, <mergePolicy> is deprecated, use <mergePolicyFactory> instead.
   [junit4]   2> 1484939 INFO  (coreLoadExecutor-2198-thread-1) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 1484945 INFO  (coreLoadExecutor-2198-thread-1) [    ] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1484956 INFO  (coreLoadExecutor-2198-thread-1) [    ] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 1484967 INFO  (coreLoadExecutor-2198-thread-1) [    ] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 1484969 INFO  (coreLoadExecutor-2198-thread-1) [    ] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properties: Can't find resource 'configsetprops.json' in classpath or '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/collection1'
   [junit4]   2> 1484969 INFO  (coreLoadExecutor-2198-thread-1) [    ] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/./collection1
   [junit4]   2> 1484969 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1484969 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/./collection1/data/]
   [junit4]   2> 1484969 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.SolrCore JMX monitoring not detected for core: collection1
   [junit4]   2> 1484969 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/./collection1/data/snapshot_metadata
   [junit4]   2> 1484970 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.s.SolrSnapshotMetaDataManager Loading from snapshot metadata file...
   [junit4]   2> 1484982 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/./collection1/data
   [junit4]   2> 1484986 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.SolrCore New index directory detected: old=null new=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/./collection1/data/index/
   [junit4]   2> 1484986 WARN  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1] Solr index directory '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/./collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1484986 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/./collection1/data/index
   [junit4]   2> 1484987 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=23, maxMergeAtOnceExplicit=38, maxMergedSegmentMB=12.0078125, floorSegmentMB=0.3564453125, forceMergeDeletesPctAllowed=7.7738484720713386, segmentsPerTier=10.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 1484994 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@433a7d2 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5b6d9402; maxCacheMB=0.7265625 maxMergeSizeMB=0.3115234375)),segFN=segments_1,generation=1}
   [junit4]   2> 1484994 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 1485124 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1485126 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1485127 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1485127 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1485128 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 1485129 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.PluginBag requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 1485129 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,standard,lazy,/update/json/docs,/admin/luke,/export,/get,/admin/properties,/update/json,/admin/threads,/update/csv,/sql,/graph,/admin/segments,/admin/system,/replication,defaults,/config,/stream,/schema,/admin/plugins,/admin/logging,/admin/ping,/update,/admin/file,/terms
   [junit4]   2> 1485129 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.SolrCore Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1485130 INFO  (OldIndexDirectoryCleanupThreadForCore-collection1) [    x:collection1] o.a.s.c.SolrCore Looking for old index directories to cleanup for core collection1 in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/./collection1/data/
   [junit4]   2> 1485130 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 1485130 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 1485130 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=2, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4865483743754896]
   [junit4]   2> 1485131 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@433a7d2 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5b6d9402; maxCacheMB=0.7265625 maxMergeSizeMB=0.3115234375)),segFN=segments_1,generation=1}
   [junit4]   2> 1485131 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 1485131 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@10cf9d91[collection1] main]
   [junit4]   2> 1485131 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage File-based storage initialized to use dir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/collection1/conf
   [junit4]   2> 1485131 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.r.RestManager Initializing RestManager with initArgs: {storageDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/collection1/conf}
   [junit4]   2> 1485131 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using file:dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-001/collection1/conf
   [junit4]   2> 1485132 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 1485132 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Replication enabled for following config files: schema.xml,xslt/dummy.xsl
   [junit4]   2> 1485132 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 1485132 INFO  (coreLoadExecutor-2198-thread-1) [    x:collection1] o.a.s.c.CoreContainer registering core: collection1
   [junit4]   2> 1485133 INFO  (searcherExecutor-2199-thread-1-processing-x:collection1) [    x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@10cf9d91[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1485664 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-002/collection1
   [junit4]   2> 1485667 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 1485671 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7cfc65c{/solr,null,AVAILABLE}
   [junit4]   2> 1485671 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.e.j.s.ServerConnector Started ServerConnector@56b80777{HTTP/1.1,[http/1.1]}{127.0.0.1:34325}
   [junit4]   2> 1485671 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.e.j.s.Server Started @1493926ms
   [junit4]   2> 1485671 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-002/collection1/data, hostContext=/solr, hostPort=34325}
   [junit4]   2> 1485672 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 1485672 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_8EB1B6904EB41305-001/solr-instance-002'
   [junit4]   2> 1485672 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1485672 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 1485678 INFO  (TEST-TestReplicationHandler.testEmptyCommits-seed#[8EB1B6904EB41305]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handle

[...truncated too long message...]

a{/,null,UNAVAILABLE}
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=DistributedFacetPivotLongTailTest -Dtests.method=test -Dtests.seed=8EB1B6904EB41305 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=ar-SA -Dtests.timezone=Etc/Universal -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR    115s J2 | DistributedFacetPivotLongTailTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:49359//collection1: Expected mime type application/octet-stream but got text/html. <html>
   [junit4]    > <head>
   [junit4]    > <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
   [junit4]    > <title>Error 500 </title>
   [junit4]    > </head>
   [junit4]    > <body>
   [junit4]    > <h2>HTTP ERROR: 500</h2>
   [junit4]    > <p>Problem accessing /collection1/update. Reason:
   [junit4]    > <pre>    java.lang.OutOfMemoryError: GC overhead limit exceeded</pre></p>
   [junit4]    > <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
   [junit4]    > </body>
   [junit4]    > </html>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([8EB1B6904EB41305:6E5894AE0487EFD]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:152)
   [junit4]    > 	at org.apache.solr.handler.component.DistributedFacetPivotLongTailTest.test(DistributedFacetPivotLongTailTest.java:71)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 7842380 INFO  (SUITE-DistributedFacetPivotLongTailTest-seed#[8EB1B6904EB41305]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.handler.component.DistributedFacetPivotLongTailTest_8EB1B6904EB41305-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {stat_i=PostingsFormat(name=Direct), range_facet_l_dv=PostingsFormat(name=LuceneVarGapDocFreqInterval), multiDefault=BlockTreeOrds(blocksize=128), intDefault=PostingsFormat(name=Direct), foo_s=PostingsFormat(name=Direct), id=PostingsFormat(name=LuceneVarGapDocFreqInterval), range_facet_i_dv=PostingsFormat(name=Direct), range_facet_l=PostingsFormat(name=Direct), timestamp=PostingsFormat(name=Direct)}, docValues:{range_facet_l_dv=DocValuesFormat(name=Lucene54), range_facet_i_dv=DocValuesFormat(name=Memory), timestamp=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=1307, maxMBSortInHeap=6.210753653059481, sim=ClassicSimilarity, locale=ar-SA, timezone=Etc/Universal
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=133801976,total=477626368
   [junit4]   2> NOTE: All tests run in this JVM: [TestPerFieldSimilarityWithDefaultOverride, MultiThreadedOCPTest, RecoveryAfterSoftCommitTest, TestDistributedMissingSort, TestLFUCache, TestTolerantSearch, TestSolrCloudWithSecureImpersonation, HdfsThreadLeakTest, HardAutoCommitTest, HdfsCollectionsAPIDistributedZkTest, TestAuthorizationFramework, TestConfigSetImmutable, TestSolr4Spatial2, TestXIncludeConfig, ForceLeaderTest, DirectUpdateHandlerOptimizeTest, TestFilteredDocIdSet, TestJmxMonitoredMap, TestConfig, SolrCLIZkUtilsTest, TestReversedWildcardFilterFactory, TestDistribIDF, TestRequestStatusCollectionAPI, TestCoreDiscovery, TestFreeTextSuggestions, TermVectorComponentDistributedTest, TestSolrDeletionPolicy1, TestExtendedDismaxParser, TestMergePolicyConfig, AutoCommitTest, DeleteStatusTest, BitVectorTest, FullSolrCloudDistribCmdsTest, TestCollationFieldDocValues, TestDynamicFieldCollectionResource, TestAnalyzeInfixSuggestions, TestStressReorder, ResponseLogComponentTest, BaseCdcrDistributedZkTest, SpellPossibilityIteratorTest, XmlUpdateRequestHandlerTest, TestStressUserVersions, TestUtils, TestOnReconnectListenerSupport, AnalyticsMergeStrategyTest, AddSchemaFieldsUpdateProcessorFactoryTest, DistributedFacetPivotSmallTest, SSLMigrationTest, ShowFileRequestHandlerTest, ZkStateReaderTest, TestPostingsSolrHighlighter, BlockDirectoryTest, HdfsLockFactoryTest, TestSchemaManager, GraphQueryTest, HdfsBasicDistributedZkTest, UnloadDistributedZkTest, TestMiniSolrCloudCluster, TestSizeLimitedDistributedMap, TestOrdValues, TestDynamicLoading, TimeZoneUtilsTest, JSONWriterTest, TestClassicSimilarityFactory, TestCryptoKeys, UpdateParamsTest, CurrencyFieldXmlFileTest, BadIndexSchemaTest, TestSchemaNameResource, TestCollationField, StatelessScriptUpdateProcessorFactoryTest, CollectionsAPIAsyncDistributedZkTest, TestGroupingSearch, TestSolrQueryParser, TestShardHandlerFactory, TestBinaryField, SuggesterTest, TestManagedSynonymFilterFactory, TestExclusionRuleCollectionAccess, TolerantUpdateProcessorTest, TestReRankQParserPlugin, OpenExchangeRatesOrgProviderTest, PreAnalyzedFieldTest, SimpleFacetsTest, PrimUtilsTest, DistribJoinFromCollectionTest, PluginInfoTest, PingRequestHandlerTest, TestCloudPivotFacet, TestAddFieldRealTimeGet, TestBlobHandler, DistributedDebugComponentTest, SuggesterTSTTest, TestSubQueryTransformerCrossCore, TestQueryWrapperFilter, TestZkChroot, TestReload, TestSort, CdcrUpdateLogTest, FieldMutatingUpdateProcessorTest, LeaderElectionTest, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, CollectionsAPIDistributedZkTest, SyncSliceTest, LeaderElectionIntegrationTest, ClusterStateUpdateTest, ZkSolrClientTest, TestRecovery, TestHashPartitioner, DistributedTermsComponentTest, TestRangeQuery, TestSolrDeletionPolicy2, MultiTermTest, MinimalSchemaTest, TestElisionMultitermQuery, OutputWriterTest, TestFuzzyAnalyzedSuggestions, TestSolrCoreProperties, DirectSolrConnectionTest, NotRequiredUniqueKeyTest, TestLuceneMatchVersion, TestPhraseSuggestions, SynonymTokenizerTest, TestDFRSimilarityFactory, TestPerFieldSimilarity, TestLMJelinekMercerSimilarityFactory, ResourceLoaderTest, ScriptEngineTest, URLClassifyProcessorTest, TestFastLRUCache, DateMathParserTest, ChaosMonkeyNothingIsSafeTest, TestSuggestSpellingConverter, DateFieldTest, DOMUtilTest, TestSolrJ, UUIDFieldTest, FileUtilsTest, CircularListTest, DistributedMLTComponentTest, TestRTGBase, SolrTestCaseJ4Test, TestCrossCoreJoin, TestCursorMarkWithoutUniqueKey, TestHighlightDedupGrouping, TestEmbeddedSolrServerConstructors, TestJettySolrRunner, AliasIntegrationTest, CdcrBootstrapTest, CdcrVersionReplicationTest, CleanupOldIndexTest, CloudExitableDirectoryReaderTest, CollectionReloadTest, CollectionsAPISolrJTest, DeleteInactiveReplicaTest, DeleteLastCustomShardedReplicaTest, DeleteNodeTest, DeleteReplicaTest, LeaderFailoverAfterPartitionTest, LeaderInitiatedRecoveryOnShardRestartTest, SharedFSAutoReplicaFailoverTest, SimpleCollectionCreateDeleteTest, TestAuthenticationFramework, TestCloudDeleteByQuery, TestCloudPseudoReturnFields, TestCollectionAPI, TestConfigSetsAPI, TestConfigSetsAPIExclusivity, TestLeaderElectionZkExpiry, TestLeaderInitiatedRecoveryThread, TestLockTree, TestRandomFlRTGCloud, TestTolerantUpdateProcessorCloud, TestTolerantUpdateProcessorRandomCloud, TriLevelCompositeIdRoutingTest, HdfsRecoverLeaseTest, HdfsRestartWhileUpdatingTest, HdfsWriteToMultipleCollectionsTest, CachingDirectoryFactoryTest, RequestLoggingTest, TestConfigReload, CoreAdminCreateDiscoverTest, CoreAdminRequestStatusTest, InfoHandlerTest, SegmentsInfoRequestHandlerTest, DistributedFacetExistsSmallTest, DistributedFacetPivotLargeTest, DistributedFacetPivotLongTailTest]
   [junit4] Completed [532/631 (7!)] on J2 in 114.63s, 1 test, 1 error <<< FAILURES!

[...truncated 314 lines...]
   [junit4] JVM J2: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20160920_092105_519.sysout
   [junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid13092.hprof ...
   [junit4] Heap dump file created [484414264 bytes in 5.788 secs]
   [junit4] <<< JVM J2: EOF ----

[...truncated 11041 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:715: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid13092.hprof

Total time: 419 minutes 6 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1118 - Still Unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1118/

18 tests failed.
FAILED:  org.apache.lucene.codecs.memory.TestDirectDocValuesFormat.testByteNumericsVsStoredFields

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([9FA0910DF6187A3D]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.codecs.memory.TestDirectDocValuesFormat

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([9FA0910DF6187A3D]:0)


FAILED:  org.apache.lucene.search.TestFuzzyQuery.testRandom

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([3ED1EDC594C2CDD4]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.search.TestFuzzyQuery

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([3ED1EDC594C2CDD4]:0)


FAILED:  org.apache.solr.TestDistributedSearch.test

Error Message:
Captured an uncaught exception in thread: Thread[id=50587, name=Thread-37029, state=RUNNABLE, group=TGRP-TestDistributedSearch]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=50587, name=Thread-37029, state=RUNNABLE, group=TGRP-TestDistributedSearch]
	at __randomizedtesting.SeedInfo.seed([320F8EB76642CB24:BA5BB16DC8BEA6DC]:0)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:39243//collection1: java.lang.NullPointerException
	at org.apache.solr.search.grouping.distributed.responseprocessor.StoredFieldsShardResponseProcessor.process(StoredFieldsShardResponseProcessor.java:41)
	at org.apache.solr.handler.component.QueryComponent.handleGroupedResponses(QueryComponent.java:751)
	at org.apache.solr.handler.component.QueryComponent.handleResponses(QueryComponent.java:734)
	at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:422)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:154)
	at org.apache.solr.core.SolrCore.execute(SolrCore.java:2107)
	at org.apache.solr.servlet.HttpSolrCall.execute(HttpSolrCall.java:653)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:460)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(Thread.java:745)

	at __randomizedtesting.SeedInfo.seed([320F8EB76642CB24]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:608)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
	at org.apache.solr.TestDistributedSearch$1.run(TestDistributedSearch.java:1126)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationStartStop

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard1

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard1
	at __randomizedtesting.SeedInfo.seed([320F8EB76642CB24:B1CC5024729A40AD]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationStartStop(CdcrReplicationDistributedZkTest.java:167)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable

Error Message:
Timeout while trying to assert replication errors

Stack Trace:
java.lang.AssertionError: Timeout while trying to assert replication errors
	at __randomizedtesting.SeedInfo.seed([320F8EB76642CB24:D49BD4C46D1DA60C]:0)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable(CdcrReplicationDistributedZkTest.java:145)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.AssertionError
	at org.junit.Assert.fail(Assert.java:92)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertTrue(Assert.java:54)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable(CdcrReplicationDistributedZkTest.java:132)
	... 41 more


FAILED:  org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest.test

Error Message:
Error from server at http://127.0.0.1:55445/oneInstanceCollection24: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /oneInstanceCollection24/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.OutOfMemoryError},msg=SolrCore &apos;oneInstanceCollection24&apos; is not available due to init failure: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.,trace=org.apache.solr.common.SolrException: SolrCore &apos;oneInstanceCollection24&apos; is not available due to init failure: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1117)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:252)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:414)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.solr.common.SolrException: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.  at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:833)  at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:706)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:862)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:799)  at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:85)  at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:374)  at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:365)  at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:156)  at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:154)  at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:659)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:441)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)  ... 12 more Caused by: java.lang.RuntimeException: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.  at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:304)  at org.apache.solr.core.HdfsDirectoryFactory.getBlockDirectoryCache(HdfsDirectoryFactory.java:280)  at org.apache.solr.core.HdfsDirectoryFactory.create(HdfsDirectoryFactory.java:220)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:346)  at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:66)  at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:604)  at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:780)  ... 35 more Caused by: java.lang.OutOfMemoryError: Direct buffer memory  at java.nio.Bits.reserveMemory(Bits.java:693)  at java.nio.DirectByteBuffer.&lt;init&gt;(DirectByteBuffer.java:123)  at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)  at org.apache.solr.store.blockcache.BlockCache.&lt;init&gt;(BlockCache.java:63)  at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:302)  ... 41 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:55445/oneInstanceCollection24: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /oneInstanceCollection24/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.OutOfMemoryError},msg=SolrCore &apos;oneInstanceCollection24&apos; is not available due to init failure: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.,trace=org.apache.solr.common.SolrException: SolrCore &apos;oneInstanceCollection24&apos; is not available due to init failure: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1117)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:252)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:414)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.common.SolrException: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.
	at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:833)
	at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:706)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:862)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:799)
	at org.apache.solr.handler.admin.CoreAdminOperation.lambda$static$0(CoreAdminOperation.java:85)
	at org.apache.solr.handler.admin.CoreAdminOperation.execute(CoreAdminOperation.java:374)
	at org.apache.solr.handler.admin.CoreAdminHandler$CallInfo.call(CoreAdminHandler.java:365)
	at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:156)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:154)
	at org.apache.solr.servlet.HttpSolrCall.handleAdminRequest(HttpSolrCall.java:659)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:441)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:462)
	... 12 more
Caused by: java.lang.RuntimeException: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.
	at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:304)
	at org.apache.solr.core.HdfsDirectoryFactory.getBlockDirectoryCache(HdfsDirectoryFactory.java:280)
	at org.apache.solr.core.HdfsDirectoryFactory.create(HdfsDirectoryFactory.java:220)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:346)
	at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:66)
	at org.apache.solr.core.SolrCore.initIndex(SolrCore.java:604)
	at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:780)
	... 35 more
Caused by: java.lang.OutOfMemoryError: Direct buffer memory
	at java.nio.Bits.reserveMemory(Bits.java:693)
	at java.nio.DirectByteBuffer.&lt;init&gt;(DirectByteBuffer.java:123)
	at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)
	at org.apache.solr.store.blockcache.BlockCache.&lt;init&gt;(BlockCache.java:63)
	at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:302)
	... 41 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([320F8EB76642CB24:BA5BB16DC8BEA6DC]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:152)
	at org.apache.solr.cloud.BasicDistributedZkTest.testANewCollectionInOneInstanceWithManualShardAssignement(BasicDistributedZkTest.java:807)
	at org.apache.solr.cloud.BasicDistributedZkTest.test(BasicDistributedZkTest.java:362)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest

Error Message:
4 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest:     1) Thread[id=61761, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=61757, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=61763, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=61759, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 4 threads leaked from SUITE scope at org.apache.solr.cloud.hdfs.HdfsBasicDistributedZkTest: 
   1) Thread[id=61761, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=61757, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=61763, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=61759, name=Connection evictor, state=TIMED_WAITING, group=TGRP-HdfsBasicDistributedZkTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([320F8EB76642CB24]:0)


FAILED:  org.apache.solr.cloud.hdfs.HdfsChaosMonkeyNothingIsSafeTest.test

Error Message:
shard1 is not consistent.  Got 107 from http://127.0.0.1:50652/c/nt/collection1 (previous client) and got 105 from http://127.0.0.1:41516/c/nt/collection1

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 107 from http://127.0.0.1:50652/c/nt/collection1 (previous client) and got 105 from http://127.0.0.1:41516/c/nt/collection1
	at __randomizedtesting.SeedInfo.seed([320F8EB76642CB24:BA5BB16DC8BEA6DC]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1271)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:231)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest.test

Error Message:
Error from server at https://127.0.0.1:38529/collection1: Expected mime type application/octet-stream but got text/html. <html> <head> <meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/> <title>Error 500 </title> </head> <body> <h2>HTTP ERROR: 500</h2> <p>Problem accessing /collection1/update. Reason: <pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.OutOfMemoryError},msg=SolrCore &apos;collection1&apos; is not available due to init failure: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.  at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1117)  at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:252)  at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:414)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)  at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)  at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)  at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)  at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)  at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)  at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)  at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)  at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)  at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)  at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.eclipse.jetty.server.Server.handle(Server.java:518)  at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)  at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)  at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)  at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)  at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)  at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)  at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)  at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)  at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.solr.common.SolrException: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.  at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:833)  at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:706)  at org.apache.solr.core.CoreContainer.create(CoreContainer.java:862)  at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:518)  at java.util.concurrent.FutureTask.run(FutureTask.java:266)  at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)  at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)  at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)  ... 1 more Caused by: java.lang.RuntimeException: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.  at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:304)  at org.apache.solr.core.HdfsDirectoryFactory.getBlockDirectoryCache(HdfsDirectoryFactory.java:280)  at org.apache.solr.core.HdfsDirectoryFactory.create(HdfsDirectoryFactory.java:220)  at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:346)  at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:426)  at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:774)  ... 8 more Caused by: java.lang.OutOfMemoryError: Direct buffer memory  at java.nio.Bits.reserveMemory(Bits.java:693)  at java.nio.DirectByteBuffer.&lt;init&gt;(DirectByteBuffer.java:123)  at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)  at org.apache.solr.store.blockcache.BlockCache.&lt;init&gt;(BlockCache.java:63)  at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:302)  ... 13 more ,code=500}</pre></p> <hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/> </body> </html> 

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:38529/collection1: Expected mime type application/octet-stream but got text/html. <html>
<head>
<meta http-equiv="Content-Type" content="text/html;charset=ISO-8859-1"/>
<title>Error 500 </title>
</head>
<body>
<h2>HTTP ERROR: 500</h2>
<p>Problem accessing /collection1/update. Reason:
<pre>    {metadata={error-class=org.apache.solr.common.SolrException,root-error-class=java.lang.OutOfMemoryError},msg=SolrCore &apos;collection1&apos; is not available due to init failure: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.,trace=org.apache.solr.common.SolrException: SolrCore &apos;collection1&apos; is not available due to init failure: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.
	at org.apache.solr.core.CoreContainer.getCore(CoreContainer.java:1117)
	at org.apache.solr.servlet.HttpSolrCall.init(HttpSolrCall.java:252)
	at org.apache.solr.servlet.HttpSolrCall.call(HttpSolrCall.java:414)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:257)
	at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:208)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:111)
	at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1676)
	at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:581)
	at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:224)
	at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1160)
	at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:511)
	at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:185)
	at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1092)
	at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)
	at org.eclipse.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:399)
	at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)
	at org.eclipse.jetty.server.Server.handle(Server.java:518)
	at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:308)
	at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:244)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.ssl.SslConnection.onFillable(SslConnection.java:186)
	at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:273)
	at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:95)
	at org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceAndRun(ExecuteProduceConsume.java:246)
	at org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:156)
	at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:654)
	at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:572)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.common.SolrException: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.
	at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:833)
	at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:706)
	at org.apache.solr.core.CoreContainer.create(CoreContainer.java:862)
	at org.apache.solr.core.CoreContainer.lambda$load$0(CoreContainer.java:518)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more
Caused by: java.lang.RuntimeException: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=&lt;size&gt;g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~8.388608 MB.
	at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:304)
	at org.apache.solr.core.HdfsDirectoryFactory.getBlockDirectoryCache(HdfsDirectoryFactory.java:280)
	at org.apache.solr.core.HdfsDirectoryFactory.create(HdfsDirectoryFactory.java:220)
	at org.apache.solr.core.CachingDirectoryFactory.get(CachingDirectoryFactory.java:346)
	at org.apache.solr.core.SolrCore.initSnapshotMetaDataManager(SolrCore.java:426)
	at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:774)
	... 8 more
Caused by: java.lang.OutOfMemoryError: Direct buffer memory
	at java.nio.Bits.reserveMemory(Bits.java:693)
	at java.nio.DirectByteBuffer.&lt;init&gt;(DirectByteBuffer.java:123)
	at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)
	at org.apache.solr.store.blockcache.BlockCache.&lt;init&gt;(BlockCache.java:63)
	at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:302)
	... 13 more
,code=500}</pre></p>
<hr /><a href="http://eclipse.org/jetty">Powered by Jetty:// 9.3.8.v20160314</a><hr/>
</body>
</html>

	at __randomizedtesting.SeedInfo.seed([320F8EB76642CB24:BA5BB16DC8BEA6DC]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:576)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:895)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:858)
	at org.apache.solr.client.solrj.SolrClient.deleteByQuery(SolrClient.java:873)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.del(AbstractFullDistribZkTestBase.java:835)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.tryDelete(ChaosMonkeySafeLeaderTest.java:190)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:92)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:56594/ypv

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:56594/ypv
	at __randomizedtesting.SeedInfo.seed([320F8EB76642CB24:BA5BB16DC8BEA6DC]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.UnloadDistributedZkTest.testCoreUnloadAndLeaders(UnloadDistributedZkTest.java:349)
	at org.apache.solr.cloud.UnloadDistributedZkTest.test(UnloadDistributedZkTest.java:67)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 46 more


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigAliasReplication

Error Message:
[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_320F8EB76642CB24-001/solr-instance-009/./collection1/data/index.20160919023959963, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_320F8EB76642CB24-001/solr-instance-009/./collection1/data, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_320F8EB76642CB24-001/solr-instance-009/./collection1/data/snapshot_metadata, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_320F8EB76642CB24-001/solr-instance-009/./collection1/data/index.20160919024000125] expected:<3> but was:<4>

Stack Trace:
java.lang.AssertionError: [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_320F8EB76642CB24-001/solr-instance-009/./collection1/data/index.20160919023959963, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_320F8EB76642CB24-001/solr-instance-009/./collection1/data, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_320F8EB76642CB24-001/solr-instance-009/./collection1/data/snapshot_metadata, /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_320F8EB76642CB24-001/solr-instance-009/./collection1/data/index.20160919024000125] expected:<3> but was:<4>
	at __randomizedtesting.SeedInfo.seed([320F8EB76642CB24:C57C60EFA0AA64C2]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.apache.solr.handler.TestReplicationHandler.checkForSingleIndex(TestReplicationHandler.java:902)
	at org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigAliasReplication(TestReplicationHandler.java:1334)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard1

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard1
	at __randomizedtesting.SeedInfo.seed([320F8EB76642CB24:6E126936A8627F12]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart(CdcrReplicationDistributedZkTest.java:264)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:52019

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:52019
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:516)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterLeaderChange

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard2

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard2
	at __randomizedtesting.SeedInfo.seed([320F8EB76642CB24:E0FFC25438ED6D16]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterLeaderChange(CdcrReplicationDistributedZkTest.java:306)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evalu

[...truncated too long message...]

odAccessorImpl.java:62)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.MethodMetric$2.snapshot(MethodMetric.java:111)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.MethodMetric.snapshot(MethodMetric.java:144)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.MetricsRegistry.snapshot(MetricsRegistry.java:401)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.MetricsSourceBuilder$1.getMetrics(MetricsSourceBuilder.java:79)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMetrics(MetricsSourceAdapter.java:194)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.updateJmxCache(MetricsSourceAdapter.java:172)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.getMBeanInfo(MetricsSourceAdapter.java:151)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.getClassName(DefaultMBeanServerInterceptor.java:1804)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.safeGetClassName(DefaultMBeanServerInterceptor.java:1595)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.checkMBeanPermission(DefaultMBeanServerInterceptor.java:1813)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:430)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
   [junit4]   2> 	at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
   [junit4]   2> 	at org.apache.hadoop.metrics2.util.MBeans.unregister(MBeans.java:81)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stopMBeans(MetricsSourceAdapter.java:226)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stop(MetricsSourceAdapter.java:211)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stopSources(MetricsSystemImpl.java:463)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
   [junit4]   2> 	at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
   [junit4]   2> 	at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
   [junit4]   2> 	at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:198)
   [junit4]   2> 	at org.apache.solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest.teardownClass(HdfsChaosMonkeySafeLeaderTest.java:46)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: java.lang.NullPointerException
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
   [junit4]   2> 	... 54 more
   [junit4]   2> 6626859 INFO  (SUITE-HdfsChaosMonkeySafeLeaderTest-seed#[320F8EB76642CB24]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsChaosMonkeySafeLeaderTest_320F8EB76642CB24-001
   [junit4]   2> Sep 18, 2016 7:58:00 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62), sim=RandomSimilarity(queryNorm=false): {}, locale=sl-SI, timezone=Pacific/Tahiti
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=2,free=319403528,total=524812288
   [junit4]   2> NOTE: All tests run in this JVM: [ZkCLITest, SliceStateTest, HdfsRecoverLeaseTest, CdcrVersionReplicationTest, TestHdfsBackupRestoreCore, PKIAuthenticationIntegrationTest, HdfsDirectoryFactoryTest, DistribDocExpirationUpdateProcessorTest, DocumentBuilderTest, TestHashPartitioner, HdfsDirectoryTest, TestRestoreCore, TermVectorComponentTest, HighlighterMaxOffsetTest, ReplaceNodeTest, SampleTest, DocValuesMissingTest, TestManagedSchemaAPI, TestElisionMultitermQuery, HighlighterTest, TestFileDictionaryLookup, TestCSVResponseWriter, TestMacros, CdcrReplicationDistributedZkTest, AliasIntegrationTest, SuggestComponentTest, TestDefaultSearchFieldResource, DistributedQueryComponentCustomSortTest, OpenCloseCoreStressTest, TestUseDocValuesAsStored2, SearchHandlerTest, SolrIndexConfigTest, DistributedIntervalFacetingTest, TestStandardQParsers, TestGraphMLResponseWriter, BigEndianAscendingWordSerializerTest, TestQueryUtils, BJQParserTest, TestEmbeddedSolrServerConstructors, TestOverriddenPrefixQueryForCustomFieldType, TestStressVersions, TestFieldCacheWithThreads, TestConfigOverlay, TestSurroundQueryParser, TestManagedResource, TestCloudInspectUtil, BlobRepositoryCloudTest, ZkSolrClientTest, SpatialFilterTest, ShardRoutingCustomTest, FullHLLTest, CloudExitableDirectoryReaderTest, HdfsUnloadDistributedZkTest, TestSearchPerf, TermsComponentTest, MoreLikeThisHandlerTest, SimplePostToolTest, URLClassifyProcessorTest, ClassificationUpdateProcessorFactoryTest, TestBulkSchemaAPI, SpellCheckCollatorTest, HdfsBasicDistributedZk2Test, TestSuggestSpellingConverter, MultiTermTest, SystemInfoHandlerTest, TestManagedStopFilterFactory, TestClusterStateMutator, TestShortCircuitedRequests, TestHashQParserPlugin, RollingRestartTest, ShardSplitTest, TestRTGBase, BlockJoinFacetSimpleTest, TestSchemaSimilarityResource, TestCursorMarkWithoutUniqueKey, HdfsSyncSliceTest, BasicFunctionalityTest, TestWordDelimiterFilterFactory, TestCharFilters, TestUpdate, TestFieldSortValues, PeerSyncReplicationTest, DistanceUnitsTest, TestCollectionAPI, VersionInfoTest, TestSolrCloudWithKerberosAlt, SolrIndexSplitterTest, TestExactStatsCache, TestInfoStreamLogging, DeleteLastCustomShardedReplicaTest, TestRuleBasedAuthorizationPlugin, TestDocBasedVersionConstraints, TestFieldCacheSort, TestLRUCache, SimpleCollectionCreateDeleteTest, TestCloudSchemaless, BooleanFieldTest, CreateCollectionCleanupTest, TestSweetSpotSimilarityFactory, TestRequestForwarding, BlockCacheTest, TestScoreJoinQPScore, SolrCmdDistributorTest, TestDistribDocBasedVersion, TestLockTree, TestMiniSolrCloudCluster, TestRequestStatusCollectionAPI, TestStressCloudBlindAtomicUpdates, VMParamsZkACLAndCredentialsProvidersTest, HdfsBasicDistributedZkTest, HdfsChaosMonkeyNothingIsSafeTest, HdfsChaosMonkeySafeLeaderTest]
   [junit4] Completed [502/631 (9!)] on J1 in 70.57s, 1 test, 1 error <<< FAILURES!

[...truncated 60608 lines...]



[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1117 - Still unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1117/

6 tests failed.
FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard1

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard1
	at __randomizedtesting.SeedInfo.seed([937EBF689F1DDA9C:CF6358E9513D6EAA]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart(CdcrReplicationDistributedZkTest.java:264)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:53419/j_xez/ar

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:53419/j_xez/ar
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:516)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterLeaderChange

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard2

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard2
	at __randomizedtesting.SeedInfo.seed([937EBF689F1DDA9C:418EF38BC1B27CAE]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterLeaderChange(CdcrReplicationDistributedZkTest.java:306)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:40723/pyaha

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:40723/pyaha
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:516)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
	at __randomizedtesting.SeedInfo.seed([937EBF689F1DDA9C]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.GeneratedMethodAccessor22.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest:     1) Thread[id=132659, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=132699, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=132704, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=132664, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=132674, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=132679, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=132684, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=132669, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=132694, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=132689, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 
   1) Thread[id=132659, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=132699, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=132704, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=132664, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=132674, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=132679, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=132684, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=132669, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=132694, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=132689, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([937EBF689F1DDA9C]:0)




Build Log:
[...truncated 11697 lines...]
   [junit4] Suite: org.apache.solr.cloud.CdcrReplicationDistributedZkTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/init-core-data-001
   [junit4]   2> 501069 INFO  (SUITE-CdcrReplicationDistributedZkTest-seed#[937EBF689F1DDA9C]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 501069 INFO  (SUITE-CdcrReplicationDistributedZkTest-seed#[937EBF689F1DDA9C]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /hl/xk
   [junit4]   2> 501090 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 501095 INFO  (Thread-4194) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 501095 INFO  (Thread-4194) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 501197 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.ZkTestServer start zk server on port:54824
   [junit4]   2> 501211 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 501215 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 501229 INFO  (zkCallback-211-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@114a745b name:ZooKeeperConnection Watcher:127.0.0.1:54824 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 501229 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 501230 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 501230 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 501233 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 501234 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 501251 INFO  (zkCallback-212-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@27746dde name:ZooKeeperConnection Watcher:127.0.0.1:54824/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 501251 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 501252 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 501252 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1
   [junit4]   2> 501256 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/shards
   [junit4]   2> 501257 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection
   [junit4]   2> 501258 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection/shards
   [junit4]   2> 501260 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-cdcr.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 501260 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 501261 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 501262 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/schema.xml
   [junit4]   2> 501267 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 501268 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 501269 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 501269 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 501270 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 501270 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/protwords.txt
   [junit4]   2> 501272 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 501272 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/currency.xml
   [junit4]   2> 501273 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 501273 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 501279 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 501280 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 501281 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 501281 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 501282 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 501282 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 501284 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 501284 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 501288 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testBatchAddsWithDelete
   [junit4]   2> 501408 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores/collection1
   [junit4]   2> 501409 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 501412 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@5282d201{/hl/xk,null,AVAILABLE}
   [junit4]   2> 501413 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.e.j.s.ServerConnector Started ServerConnector@4082e43a{HTTP/1.1,[http/1.1]}{127.0.0.1:43328}
   [junit4]   2> 501413 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.e.j.s.Server Started @510644ms
   [junit4]   2> 501413 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {shards=shard1, hostContext=/hl/xk, hostPort=43328, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores}
   [junit4]   2> 501414 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 501414 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001'
   [junit4]   2> 501414 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 501414 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 501414 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 501415 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 501427 INFO  (zkCallback-213-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@52ce4a5a name:ZooKeeperConnection Watcher:127.0.0.1:54824/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 501427 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 501427 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 501429 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 501429 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/solr.xml
   [junit4]   2> 501434 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores
   [junit4]   2> 501434 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.CoreContainer New CoreContainer 2039324192
   [junit4]   2> 501435 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001]
   [junit4]   2> 501435 WARN  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.CoreContainer Couldn't add files from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/lib to classpath: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/lib
   [junit4]   2> 501447 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,
   [junit4]   2> 501455 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.u.UpdateShardHandler Created UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&maxConnections=100000&maxConnectionsPerHost=100000
   [junit4]   2> 501455 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 501455 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 501455 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:54824/solr
   [junit4]   2> 501455 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 501455 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 501463 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 501479 INFO  (zkCallback-216-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3ba14d60 name:ZooKeeperConnection Watcher:127.0.0.1:54824 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 501495 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 501495 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 501507 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$12/1549056544@2ff19dd2
   [junit4]   2> 501508 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 501516 INFO  (zkCallback-217-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4add716 name:ZooKeeperConnection Watcher:127.0.0.1:54824/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 501516 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 501520 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
   [junit4]   2> 501521 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
   [junit4]   2> 501524 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
   [junit4]   2> 501525 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
   [junit4]   2> 501527 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
   [junit4]   2> 501528 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 501529 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 501530 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 501538 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 501539 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 501539 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 501540 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 501542 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
   [junit4]   2> 501543 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
   [junit4]   2> 501544 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 501544 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/96600495612559364-127.0.0.1:43328_hl%2Fxk-n_0000000000
   [junit4]   2> 501545 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:43328_hl%2Fxk
   [junit4]   2> 501545 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
   [junit4]   2> 501546 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.Overseer Overseer (id=96600495612559364-127.0.0.1:43328_hl%2Fxk-n_0000000000) starting
   [junit4]   2> 501547 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
   [junit4]   2> 501556 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=10000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 501575 INFO  (OverseerStateUpdate-96600495612559364-127.0.0.1:43328_hl%2Fxk-n_0000000000) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.Overseer Starting to work on the main queue
   [junit4]   2> 501576 INFO  (OverseerCollectionConfigSetProcessor-96600495612559364-127.0.0.1:43328_hl%2Fxk-n_0000000000) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
   [junit4]   2> 501586 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:43328_hl%2Fxk
   [junit4]   2> 501586 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:43328_hl%2Fxk
   [junit4]   2> 501587 INFO  (OverseerStateUpdate-96600495612559364-127.0.0.1:43328_hl%2Fxk-n_0000000000) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 501595 INFO  (zkCallback-217-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 501596 INFO  (zkCallback-217-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 501637 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 501637 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 501638 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores
   [junit4]   2> 501640 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.json, collection=tmp_collection, coreNodeName=, dataDir=data/, shard=}
   [junit4]   2> 501640 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.CorePropertiesLocator Found core collection1 in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores/collection1
   [junit4]   2> 501640 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions
   [junit4]   2> 501643 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.ZkController publishing state=down
   [junit4]   2> 501643 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.ZkController numShards not found on descriptor - reading it from system property
   [junit4]   2> 501645 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.ZkController Collection tmp_collection not visible yet, but flagging it so a watch is registered when it becomes visible
   [junit4]   2> 501646 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.c.ZkStateReader Deleting data for [tmp_collection]
   [junit4]   2> 501647 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.s.SolrDispatchFilter user.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1
   [junit4]   2> 501647 INFO  (TEST-CdcrReplicationDistributedZkTest.testBatchAddsWithDelete-seed#[937EBF689F1DDA9C]) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 501648 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.ZkController look for our core node name
   [junit4]   2> 501863 INFO  (OverseerStateUpdate-96600495612559364-127.0.0.1:43328_hl%2Fxk-n_0000000000) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:43328/hl/xk",
   [junit4]   2>   "node_name":"127.0.0.1:43328_hl%2Fxk",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":null,
   [junit4]   2>   "collection":"tmp_collection",
   [junit4]   2>   "operation":"state"} current state version: 0
   [junit4]   2> 501863 INFO  (OverseerStateUpdate-96600495612559364-127.0.0.1:43328_hl%2Fxk-n_0000000000) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.o.ReplicaMutator Update state numShards=2 message={
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"http://127.0.0.1:43328/hl/xk",
   [junit4]   2>   "node_name":"127.0.0.1:43328_hl%2Fxk",
   [junit4]   2>   "numShards":"2",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":null,
   [junit4]   2>   "collection":"tmp_collection",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 501866 INFO  (OverseerStateUpdate-96600495612559364-127.0.0.1:43328_hl%2Fxk-n_0000000000) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.o.ClusterStateMutator building a new cName: tmp_collection
   [junit4]   2> 501866 INFO  (OverseerStateUpdate-96600495612559364-127.0.0.1:43328_hl%2Fxk-n_0000000000) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard2
   [junit4]   2> 501871 INFO  (zkCallback-217-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 502651 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.ZkController waiting to find shard id in clusterstate for collection1
   [junit4]   2> 502651 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.ZkController Check for collection zkNode:tmp_collection
   [junit4]   2> 502651 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.ZkController Creating collection in ZooKeeper:tmp_collection
   [junit4]   2> 502652 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.ZkController Looking for collection configName
   [junit4]   2> 502652 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.ZkController Only one config set found in zk - using it:conf1
   [junit4]   2> 502652 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.c.SolrZkClient makePath: /collections/tmp_collection
   [junit4]   2> 502653 INFO  (zkCallback-217-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 502654 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.c.ZkStateReader Load collection config from: [/collections/tmp_collection]
   [junit4]   2> 502654 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.c.ZkStateReader path=[/collections/tmp_collection] [configName]=[conf1] specified config exists in ZooKeeper
   [junit4]   2> 502654 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores/collection1'
   [junit4]   2> 502654 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 502654 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 502669 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.Config loaded config solrconfig.xml with version 0 
   [junit4]   2> 502692 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 502695 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 502727 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 502732 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 502812 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.s.IndexSchema default search field in schema is text
   [junit4]   2> 502813 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 502814 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.s.FileExchangeRateProvider Reloading exchange rates from file currency.xml
   [junit4]   2> 502818 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.s.FileExchangeRateProvider Reloading exchange rates from file currency.xml
   [junit4]   2> 502841 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properties: Can't find resource 'configsetprops.json' in classpath or '/configs/conf1', cwd=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1
   [junit4]   2> 502841 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection   x:collection1] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from collection tmp_collection
   [junit4]   2> 502841 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 502841 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores/collection1/data/]
   [junit4]   2> 502841 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.JmxMonitoredMap JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@372ca662
   [junit4]   2> 502842 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores/collection1/data/snapshot_metadata
   [junit4]   2> 502842 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.s.SolrSnapshotMetaDataManager Loading from snapshot metadata file...
   [junit4]   2> 502842 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores/collection1/data
   [junit4]   2> 502842 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore New index directory detected: old=null new=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores/collection1/data/index/
   [junit4]   2> 502842 WARN  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Solr index directory '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 502842 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores/collection1/data/index
   [junit4]   2> 502843 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=46, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1387051555292735]
   [junit4]   2> 502843 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@73f11028 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@36427ede; maxCacheMB=0.205078125 maxMergeSizeMB=0.943359375)),segFN=segments_1,generation=1}
   [junit4]   2> 502843 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 502846 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.u.p.UpdateRequestProcessorChain creating updateRequestProcessorChain "cdcr-processor-chain"
   [junit4]   2> 502857 INFO  (OldIndexDirectoryCleanupThreadForCore-collection1) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore Looking for old index directories to cleanup for core collection1 in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores/collection1/data/
   [junit4]   2> 502857 WARN  (OldIndexDirectoryCleanupThreadForCore-collection1) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.DirectoryFactory /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CdcrReplicationDistributedZkTest_937EBF689F1DDA9C-001/jetty-001/cores/collection1/data/ does not point to a valid data directory; skipping clean-up of old index directories.
   [junit4]   2> 502866 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 502867 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 502868 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 502869 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 502870 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 502882 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,standard,/update/json/docs,/admin/luke,/export,/get,/admin/properties,/update/json,/admin/threads,/update/csv,/sql,/graph,/admin/segments,/admin/system,/replication,/config,/stream,/schema,/admin/plugins,/admin/logging,/admin/ping,/cdcr,/update,/admin/file,/terms
   [junit4]   2> 502882 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 502894 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateHandler Using UpdateLog implementation: org.apache.solr.update.CdcrUpdateLog
   [junit4]   2> 502894 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateLog Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10 numVersionBuckets=65536
   [junit4]   2> 502894 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.u.CommitTracker Hard AutoCommit: disabled
   [junit4]   2> 502894 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.u.CommitTracker Soft AutoCommit: disabled
   [junit4]   2> 502895 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1721589593, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 502899 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@73f11028 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@36427ede; maxCacheMB=0.205078125 maxMergeSizeMB=0.943359375)),segFN=segments_1,generation=1}
   [junit4]   2> 502899 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 502900 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.s.SolrIndexSearcher Opening [Searcher@301d3483[collection1] main]
   [junit4]   2> 502900 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.c.ZkStateReader Load collection config from: [/collections/tmp_collection]
   [junit4]   2> 502901 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.c.ZkStateReader path=[/collections/tmp_collection] [configName]=[conf1] specified config exists in ZooKeeper
   [junit4]   2> 502901 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 502901 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 502901 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.r.RestManager Initializing RestManager with initArgs: {}
   [junit4]   2> 502901 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 502902 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 502902 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.r.ManagedResourceStorage Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 502902 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.r.RestManager Initializing 0 registered ManagedResources
   [junit4]   2> 502902 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.h.ReplicationHandler Commits will be reserved for  10000
   [junit4]   2> 502907 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.c.SolrZkClient makePath: /collections/tmp_collection/cdcr/state
   [junit4]   2> 502909 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.h.CdcrBufferStateManager Created znode /collections/tmp_collection/cdcr/state/buffer
   [junit4]   2> 502914 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.h.CdcrProcessStateManager Created znode /collections/tmp_collection/cdcr/state/process
   [junit4]   2> 503014 INFO  (searcherExecutor-518-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk x:collection1 s:shard2 c:tmp_collection r:core_node1) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SolrCore [collection1] Registered new searcher Searcher@301d3483[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 503015 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateLog Looking up max value of version field to seed version buckets
   [junit4]   2> 503015 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.u.VersionInfo Refreshing highest value of _version_ for 65536 version buckets from index
   [junit4]   2> 503015 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.u.VersionInfo No terms found for _version_, cannot seed version bucket highest value from index
   [junit4]   2> 503015 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateLog Could not find max version in index or recent updates, using new clock 1545607931789049856
   [junit4]   2> 503017 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.u.UpdateLog Took 1.0ms to seed version buckets with highest version 1545607931789049856
   [junit4]   2> 503017 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.ZkController watch zkdir /configs/conf1
   [junit4]   2> 503020 INFO  (coreLoadExecutor-517-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.CoreContainer registering core: collection1
   [junit4]   2> 503027 INFO  (coreZkRegister-510-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk x:collection1 s:shard2 c:tmp_collection r:core_node1) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.ZkController Register replica - core:collection1 address:http://127.0.0.1:43328/hl/xk collection:tmp_collection shard:shard2
   [junit4]   2> 503035 INFO  (coreZkRegister-510-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk x:collection1 s:shard2 c:tmp_collection r:core_node1) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.c.SolrZkClient makePath: /collections/tmp_collection/leader_elect/shard2/election
   [junit4]   2> 503087 INFO  (coreZkRegister-510-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk x:collection1 s:shard2 c:tmp_collection r:core_node1) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.LeaderElector Joined leadership election with path: /collections/tmp_collection/leader_elect/shard2/election/96600495612559364-core_node1-n_0000000000
   [junit4]   2> 503091 INFO  (coreZkRegister-510-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk x:collection1 s:shard2 c:tmp_collection r:core_node1) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Running the leader process for shard=shard2 and weAreReplacement=false and leaderVoteWait=10000
   [junit4]   2> 503106 INFO  (coreZkRegister-510-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk x:collection1 s:shard2 c:tmp_collection r:core_node1) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext Enough replicas found to continue.
   [junit4]   2> 503106 INFO  (coreZkRegister-510-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk x:collection1 s:shard2 c:tmp_collection r:core_node1) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.ShardLeaderElectionContext I may be the new leader - try and sync
   [junit4]   2> 503106 INFO  (coreZkRegister-510-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk x:collection1 s:shard2 c:tmp_collection r:core_node1) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync replicas to http://127.0.0.1:43328/hl/xk/collection1/
   [junit4]   2> 503106 INFO  (coreZkRegister-510-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk x:collection1 s:shard2 c:tmp_collection r:core_node1) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 r:core_node1 x:collection1] o.a.s.c.SyncStrategy Sync Success - now sync replicas to me
   [junit4]   2> 503106 INFO  (OverseerStateUpdate-96600495612559364-127.0.0.1:43328_hl%2Fxk-n_0000000000) [n:127.0.0.1:43328_hl%2Fxk    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "operation":"leader",
   [junit4]   2>   "shard":"shard2",
   [junit4]   2>   "collection":"tmp_collection"} current state version: 1
   [junit4]   2> 503106 INFO  (coreZkRegister-510-thread-1-processing-n:127.0.0.1:43328_hl%2Fxk x:collection1 s:shard2 c:tmp_collection r:core_node1) [n:127.0.0.1:43328_hl%2Fxk c:tmp_collection s:shard2 

[...truncated too long message...]

unit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
   [junit4]   2> 	at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
   [junit4]   2> 	at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
   [junit4]   2> 	at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:198)
   [junit4]   2> 	at org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.teardownClass(HdfsCollectionsAPIDistributedZkTest.java:52)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: java.lang.NullPointerException
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
   [junit4]   2> 	... 54 more
   [junit4]   2> 8304437 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[937EBF689F1DDA9C]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_937EBF689F1DDA9C-001
   [junit4]   2> Sep 16, 2016 8:26:29 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 34 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {other_tl1=Lucene50(blocksize=128), range_facet_l_dv=Lucene50(blocksize=128), _version_=Lucene50(blocksize=128), multiDefault=PostingsFormat(name=MockRandom), a_t=PostingsFormat(name=MockRandom), intDefault=Lucene50(blocksize=128), a_i1=PostingsFormat(name=MockRandom), id=Lucene50(blocksize=128), range_facet_i_dv=PostingsFormat(name=MockRandom), text=PostingsFormat(name=LuceneFixedGap), range_facet_l=PostingsFormat(name=MockRandom), timestamp=PostingsFormat(name=MockRandom)}, docValues:{range_facet_l_dv=DocValuesFormat(name=Lucene54), range_facet_i_dv=DocValuesFormat(name=Memory), timestamp=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=1608, maxMBSortInHeap=7.964928059876776, sim=ClassicSimilarity, locale=ro, timezone=Etc/GMT+11
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=240799904,total=516423680
   [junit4]   2> NOTE: All tests run in this JVM: [ClusterStateTest, TestSolrConfigHandler, HdfsRestartWhileUpdatingTest, ZkNodePropsTest, TestStressCloudBlindAtomicUpdates, TestReplicationHandler, TestSolrDeletionPolicy2, DistributedExpandComponentTest, OutputWriterTest, SmileWriterTest, DistributedSpellCheckComponentTest, TestTolerantUpdateProcessorRandomCloud, FastVectorHighlighterTest, HdfsChaosMonkeyNothingIsSafeTest, PolyFieldTest, TestMacroExpander, TestRecoveryHdfs, TestFieldCacheVsDocValues, TestHdfsCloudBackupRestore, TestHighlightDedupGrouping, TestAtomicUpdateErrorCases, TestRandomCollapseQParserPlugin, RankQueryTest, SecurityConfHandlerTest, RestartWhileUpdatingTest, TestNoOpRegenerator, TestSolrCloudWithDelegationTokens, StandardRequestHandlerTest, BufferStoreTest, CollectionsAPISolrJTest, TestPseudoReturnFields, TestRandomFlRTGCloud, ConnectionReuseTest, TestFunctionQuery, ReturnFieldsTest, QueryElevationComponentTest, TestFieldCacheSanityChecker, AlternateDirectoryTest, AtomicUpdatesTest, CdcrRequestHandlerTest, TestRangeQuery, ExitableDirectoryReaderTest, CdcrBootstrapTest, DistributedQueueTest, TestSSLRandomization, AnalyticsQueryTest, DistributedFacetPivotLongTailTest, TestFoldingMultitermQuery, BlockJoinFacetRandomTest, PeerSyncTest, TestScoreJoinQPNoScore, TestHdfsUpdateLog, ScriptEngineTest, TestFastWriter, TestFastLRUCache, TestRandomDVFaceting, AddBlockUpdateTest, OverseerModifyCollectionTest, TestTrieFacet, BadCopyFieldTest, QueryEqualityTest, CollectionStateFormat2Test, TestExpandComponent, TestManagedSchema, SolrCloudExampleTest, AnalysisAfterCoreReloadTest, TestComponentsName, TestDistributedSearch, TestRealTimeGet, TestJoin, UUIDFieldTest, DeleteReplicaTest, StressHdfsTest, TestQuerySenderNoQuery, TestSchemaResource, SolrRequestParserTest, TestRTimerTree, TestBinaryResponseWriter, TestCodecSupport, TestLocalFSCloudBackupRestore, NumericFieldsTest, CurrencyFieldOpenExchangeTest, DocumentAnalysisRequestHandlerTest, TestSolr4Spatial, SignatureUpdateProcessorFactoryTest, TestRawTransformer, BasicZkTest, HLLUtilTest, DistribCursorPagingTest, TestCloudDeleteByQuery, RemoteQueryErrorTest, JavabinLoaderTest, TestSolrCoreSnapshots, TestConfigSetProperties, SuggestComponentContextFilterQueryTest, DistributedFacetPivotWhiteBoxTest, TestRestManager, PreAnalyzedUpdateProcessorTest, TestJmxIntegration, TestMergePolicyConfig, TestDocSet, TestBinaryField, MinimalSchemaTest, TestFuzzyAnalyzedSuggestions, ExternalFileFieldSortTest, TestSolrCoreProperties, DirectSolrConnectionTest, NotRequiredUniqueKeyTest, TestPhraseSuggestions, SpellPossibilityIteratorTest, TestCharFilters, SynonymTokenizerTest, TestPerFieldSimilarity, TestLMDirichletSimilarityFactory, TestIBSimilarityFactory, TestJmxMonitoredMap, ResourceLoaderTest, PluginInfoTest, ChaosMonkeyNothingIsSafeTest, PrimUtilsTest, TestSuggestSpellingConverter, DateFieldTest, DOMUtilTest, TestLRUCache, TestDocumentBuilder, SliceStateTest, DistributedMLTComponentTest, TestRTGBase, CursorPagingTest, TestCrossCoreJoin, TestSimpleTrackingShardHandler, ActionThrottleTest, AliasIntegrationTest, AsyncCallRequestStatusResponseTest, BaseCdcrDistributedZkTest, CdcrReplicationHandlerTest, CdcrVersionReplicationTest, CleanupOldIndexTest, CollectionTooManyReplicasTest, CollectionsAPIAsyncDistributedZkTest, ConnectionManagerTest, DeleteInactiveReplicaTest, LeaderFailoverAfterPartitionTest, LeaderInitiatedRecoveryOnCommitTest, LeaderInitiatedRecoveryOnShardRestartTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverseerCollectionConfigSetProcessorTest, OverseerTaskQueueTest, PeerSyncReplicationTest, ReplaceNodeTest, ReplicationFactorTest, SaslZkACLProviderTest, SharedFSAutoReplicaFailoverUtilsTest, SimpleCollectionCreateDeleteTest, SolrCLIZkUtilsTest, TestCloudPivotFacet, TestCloudPseudoReturnFields, TestCollectionAPI, TestConfigSetsAPI, TestConfigSetsAPIExclusivity, TestConfigSetsAPIZkFailure, TestCryptoKeys, TestDistribDocBasedVersion, TestExclusionRuleCollectionAccess, TestLeaderInitiatedRecoveryThread, TestOnReconnectListenerSupport, TestRequestForwarding, TestShortCircuitedRequests, TestSolrCloudWithKerberosAlt, VMParamsZkACLAndCredentialsProvidersTest, HdfsBasicDistributedZk2Test, HdfsChaosMonkeySafeLeaderTest, HdfsCollectionsAPIDistributedZkTest]
   [junit4] Completed [523/631 (4!)] on J0 in 1419.64s, 1 test, 1 error <<< FAILURES!

[...truncated 60548 lines...]



[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1116 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1116/

11 tests failed.
FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testOps

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard2

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard2
	at __randomizedtesting.SeedInfo.seed([11395C375D5ACE69:D41C0D41269C5B4]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testOps(CdcrReplicationDistributedZkTest.java:464)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterLeaderChange

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard1

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard1
	at __randomizedtesting.SeedInfo.seed([11395C375D5ACE69:C3C910D403F5685B]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterLeaderChange(CdcrReplicationDistributedZkTest.java:305)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:43481

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:43481
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:516)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  org.apache.solr.cloud.CollectionsAPISolrJTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:56799

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:56799
	at __randomizedtesting.SeedInfo.seed([11395C375D5ACE69:996D63EDF3A6A391]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:415)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:367)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1280)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1050)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:992)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:166)
	at org.apache.solr.cloud.CollectionsAPISolrJTest.testAddAndDeleteReplica(CollectionsAPISolrJTest.java:310)
	at org.apache.solr.cloud.CollectionsAPISolrJTest.test(CollectionsAPISolrJTest.java:63)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 52 more


FAILED:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=168560, name=Thread-29390, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=168560, name=Thread-29390, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:57528/chvf/xm/collection1
	at __randomizedtesting.SeedInfo.seed([11395C375D5ACE69]:0)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:644)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:57528/chvf/xm/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:166)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:642)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 5 more


FAILED:  org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit

Error Message:
expected:<1> but was:<2>

Stack Trace:
java.lang.AssertionError: expected:<1> but was:<2>
	at __randomizedtesting.SeedInfo.seed([11395C375D5ACE69:E874CF98612F83E3]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.ShardSplitTest.testSplitAfterFailedSplit(ShardSplitTest.java:278)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail

Error Message:
expected:<200> but was:<404>

Stack Trace:
java.lang.AssertionError: expected:<200> but was:<404>
	at __randomizedtesting.SeedInfo.seed([11395C375D5ACE69:7986691D8DC0DC85]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.cancelDelegationToken(TestSolrCloudWithDelegationTokens.java:137)
	at org.apache.solr.cloud.TestSolrCloudWithDelegationTokens.testDelegationTokenCancelFail(TestSolrCloudWithDelegationTokens.java:282)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsTlogReplayBufferedWhileIndexingTest.test

Error Message:
There are still nodes recoverying - waited for 440 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 440 seconds
	at __randomizedtesting.SeedInfo.seed([11395C375D5ACE69:996D63EDF3A6A391]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:182)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:862)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1418)
	at org.apache.solr.cloud.TlogReplayBufferedWhileIndexingTest.test(TlogReplayBufferedWhileIndexingTest.java:99)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.handler.TestReplicationHandler

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory, NRTCachingDirectory]
	at __randomizedtesting.SeedInfo.seed([11395C375D5ACE69]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.GeneratedMethodAccessor21.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
	at __randomizedtesting.SeedInfo.seed([11395C375D5ACE69]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.GeneratedMethodAccessor95.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest:     1) Thread[id=80041, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=80021, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=80001, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=80006, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=80011, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=80026, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=80016, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=80036, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=80031, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=80046, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 
   1) Thread[id=80041, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=80021, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=80001, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=80006, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=80011, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=80026, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=80016, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=80036, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=80031, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=80046, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([11395C375D5ACE69]:0)




Build Log:
[...truncated 11203 lines...]
   [junit4] Suite: org.apache.solr.cloud.TestSolrCloudWithDelegationTokens
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_11395C375D5ACE69-001/init-core-data-001
   [junit4]   2> 379422 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[11395C375D5ACE69]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 379432 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[11395C375D5ACE69]-worker) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 379436 INFO  (Thread-559) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 379436 INFO  (Thread-559) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 379536 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[11395C375D5ACE69]-worker) [    ] o.a.s.c.ZkTestServer start zk server on port:52735
   [junit4]   2> 379536 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[11395C375D5ACE69]-worker) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 379540 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[11395C375D5ACE69]-worker) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 379584 INFO  (zkCallback-311-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@9b6aee5 name:ZooKeeperConnection Watcher:127.0.0.1:52735 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 379584 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[11395C375D5ACE69]-worker) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 379584 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[11395C375D5ACE69]-worker) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 379584 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[11395C375D5ACE69]-worker) [    ] o.a.s.c.c.SolrZkClient makePath: /solr/solr.xml
   [junit4]   2> 379601 INFO  (jetty-launcher-310-thread-1) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 379601 INFO  (jetty-launcher-310-thread-2) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 379602 INFO  (jetty-launcher-310-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@767e9948{/solr,null,AVAILABLE}
   [junit4]   2> 379603 INFO  (jetty-launcher-310-thread-1) [    ] o.e.j.s.ServerConnector Started ServerConnector@64e13eb7{HTTP/1.1,[http/1.1]}{127.0.0.1:59036}
   [junit4]   2> 379603 INFO  (jetty-launcher-310-thread-1) [    ] o.e.j.s.Server Started @387191ms
   [junit4]   2> 379603 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=59036}
   [junit4]   2> 379603 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 379603 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_11395C375D5ACE69-001/tempDir-001/node1'
   [junit4]   2> 379603 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 379603 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 379604 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 379616 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 379719 INFO  (jetty-launcher-310-thread-2) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@e3dec11{/solr,null,AVAILABLE}
   [junit4]   2> 379719 INFO  (jetty-launcher-310-thread-2) [    ] o.e.j.s.ServerConnector Started ServerConnector@11107aae{HTTP/1.1,[http/1.1]}{127.0.0.1:46719}
   [junit4]   2> 379719 INFO  (jetty-launcher-310-thread-2) [    ] o.e.j.s.Server Started @387308ms
   [junit4]   2> 379720 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=46719}
   [junit4]   2> 379720 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 379720 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_11395C375D5ACE69-001/tempDir-001/node2'
   [junit4]   2> 379720 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 379720 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 379720 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 379920 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 379948 INFO  (zkCallback-312-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@7225043d name:ZooKeeperConnection Watcher:127.0.0.1:52735/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 379948 INFO  (zkCallback-313-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@365e98ff name:ZooKeeperConnection Watcher:127.0.0.1:52735/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 379952 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 379952 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 379952 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 379952 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 379956 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 380138 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 380144 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_11395C375D5ACE69-001/tempDir-001/node2/.
   [junit4]   2> 380168 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.CoreContainer New CoreContainer 416722465
   [junit4]   2> 380168 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_11395C375D5ACE69-001/tempDir-001/node2]
   [junit4]   2> 380168 WARN  (jetty-launcher-310-thread-2) [    ] o.a.s.c.CoreContainer Couldn't add files from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_11395C375D5ACE69-001/tempDir-001/node2/lib to classpath: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_11395C375D5ACE69-001/tempDir-001/node2/lib
   [junit4]   2> 380168 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,
   [junit4]   2> 380169 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_11395C375D5ACE69-001/tempDir-001/node1/.
   [junit4]   2> 380169 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.CoreContainer New CoreContainer 1273173647
   [junit4]   2> 380170 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_11395C375D5ACE69-001/tempDir-001/node1]
   [junit4]   2> 380170 WARN  (jetty-launcher-310-thread-1) [    ] o.a.s.c.CoreContainer Couldn't add files from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_11395C375D5ACE69-001/tempDir-001/node1/lib to classpath: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_11395C375D5ACE69-001/tempDir-001/node1/lib
   [junit4]   2> 380170 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,
   [junit4]   2> 380171 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.u.UpdateShardHandler Created UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&maxConnections=100000&maxConnectionsPerHost=100000
   [junit4]   2> 380171 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 380172 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 380172 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52735/solr
   [junit4]   2> 380172 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 380172 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 380172 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 380177 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.u.UpdateShardHandler Created UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&maxConnections=100000&maxConnectionsPerHost=100000
   [junit4]   2> 380180 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 380180 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 380180 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52735/solr
   [junit4]   2> 380180 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 380180 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 380209 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 380240 INFO  (zkCallback-319-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6880baf2 name:ZooKeeperConnection Watcher:127.0.0.1:52735 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 380260 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 380261 INFO  (jetty-launcher-310-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 380262 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$90/738729067@6c0cd33b
   [junit4]   2> 380276 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 380286 INFO  (zkCallback-316-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@687cb563 name:ZooKeeperConnection Watcher:127.0.0.1:52735 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 380286 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 380287 INFO  (jetty-launcher-310-thread-2) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 380323 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$90/738729067@3a58d547
   [junit4]   2> 380372 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 380372 INFO  (zkCallback-320-thread-1-processing-n:127.0.0.1:59036_solr) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@fe3fef6 name:ZooKeeperConnection Watcher:127.0.0.1:52735/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 380372 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 380399 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
   [junit4]   2> 380416 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
   [junit4]   2> 380418 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
   [junit4]   2> 380420 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
   [junit4]   2> 380421 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
   [junit4]   2> 380423 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 380432 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 380433 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 380434 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 380435 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 380436 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 380436 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 380437 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 380439 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
   [junit4]   2> 380440 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
   [junit4]   2> 380440 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 380441 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/96591924306247685-127.0.0.1:59036_solr-n_0000000000
   [junit4]   2> 380441 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:59036_solr
   [junit4]   2> 380441 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
   [junit4]   2> 380442 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.Overseer Overseer (id=96591924306247685-127.0.0.1:59036_solr-n_0000000000) starting
   [junit4]   2> 380443 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
   [junit4]   2> 380452 INFO  (zkCallback-321-thread-1-processing-n:127.0.0.1:46719_solr) [n:127.0.0.1:46719_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3c8a1ea4 name:ZooKeeperConnection Watcher:127.0.0.1:52735/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 380452 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 380457 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 380457 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 380458 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 380468 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 380469 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/96591924306247686-127.0.0.1:46719_solr-n_0000000001
   [junit4]   2> 380469 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.LeaderElector Watching path /overseer_elect/election/96591924306247685-127.0.0.1:59036_solr-n_0000000000 to know if I could be the leader
   [junit4]   2> 380469 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:46719_solr
   [junit4]   2> 380469 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:46719_solr
   [junit4]   2> 380491 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 380491 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.CoreContainer Authentication plugin class obtained from system property 'authenticationPlugin': org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 380512 INFO  (zkCallback-320-thread-1-processing-n:127.0.0.1:59036_solr) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 380531 INFO  (zkCallback-320-thread-1-processing-n:127.0.0.1:59036_solr) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 380552 INFO  (zkCallback-321-thread-2-processing-n:127.0.0.1:46719_solr) [n:127.0.0.1:46719_solr    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 380561 INFO  (zkCallback-321-thread-2-processing-n:127.0.0.1:46719_solr) [n:127.0.0.1:46719_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 380565 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 380581 INFO  (OverseerStateUpdate-96591924306247685-127.0.0.1:59036_solr-n_0000000000) [n:127.0.0.1:59036_solr    ] o.a.s.c.Overseer Starting to work on the main queue
   [junit4]   2> 380585 INFO  (OverseerCollectionConfigSetProcessor-96591924306247685-127.0.0.1:59036_solr-n_0000000000) [n:127.0.0.1:59036_solr    ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
   [junit4]   2> 380588 INFO  (OverseerStateUpdate-96591924306247685-127.0.0.1:59036_solr-n_0000000000) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (1)
   [junit4]   2> 380602 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.ZkController Publish node=127.0.0.1:59036_solr as DOWN
   [junit4]   2> 380606 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
   [junit4]   2> 380633 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59036_solr
   [junit4]   2> 380633 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:59036_solr
   [junit4]   2> 380636 INFO  (zkCallback-321-thread-2-processing-n:127.0.0.1:46719_solr) [n:127.0.0.1:46719_solr    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 380637 INFO  (zkCallback-320-thread-1-processing-n:127.0.0.1:59036_solr) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 380637 INFO  (OverseerStateUpdate-96591924306247685-127.0.0.1:59036_solr-n_0000000000) [n:127.0.0.1:59036_solr    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "operation":"downnode",
   [junit4]   2>   "node_name":"127.0.0.1:59036_solr"} current state version: 0
   [junit4]   2> 380637 INFO  (OverseerStateUpdate-96591924306247685-127.0.0.1:59036_solr-n_0000000000) [n:127.0.0.1:59036_solr    ] o.a.s.c.o.NodeMutator DownNode state invoked for node: 127.0.0.1:59036_solr
   [junit4]   2> 380637 INFO  (zkCallback-321-thread-2-processing-n:127.0.0.1:46719_solr) [n:127.0.0.1:46719_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 380640 INFO  (zkCallback-320-thread-1-processing-n:127.0.0.1:59036_solr) [n:127.0.0.1:59036_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (2)
   [junit4]   2> 380652 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 380668 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.CoreContainer Authentication plugin class obtained from system property 'authenticationPlugin': org.apache.solr.security.HttpParamDelegationTokenPlugin
   [junit4]   2> 380668 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.s.KerberosPlugin Params: {token.valid=30, delegation-token.token-kind=solr-dt, cookie.domain=127.0.0.1, cookie.path=/, zk-dt-secret-manager.znodeWorkingPath=solr/security/zkdtsm, token.validity=36000, signer.secret.provider.zookeeper.path=/token, type=kerberos, zk-dt-secret-manager.enable=true, kerberos.name.rules=DEFAULT, signer.secret.provider=zookeeper}
   [junit4]   2> 381134 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 381154 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.c.f.i.CuratorFrameworkImpl Starting
   [junit4]   2> 381424 INFO  (jetty-launcher-310-thread-1-EventThread) [n:127.0.0.1:59036_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 381428 INFO  (jetty-launcher-310-thread-2-EventThread) [n:127.0.0.1:46719_solr    ] o.a.c.f.s.ConnectionStateManager State change: CONNECTED
   [junit4]   2> 383050 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.CoreContainer Reconfiguring HttpClient settings.
   [junit4]   2> 383051 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_11395C375D5ACE69-001/tempDir-001/node1/.
   [junit4]   2> 383051 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions
   [junit4]   2> 383052 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.s.SolrDispatchFilter user.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0
   [junit4]   2> 383053 INFO  (jetty-launcher-310-thread-1) [n:127.0.0.1:59036_solr    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 383057 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.CoreContainer Reconfiguring HttpClient settings.
   [junit4]   2> 383058 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.TestSolrCloudWithDelegationTokens_11395C375D5ACE69-001/tempDir-001/node2/.
   [junit4]   2> 383058 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions
   [junit4]   2> 383066 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.s.SolrDispatchFilter user.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0
   [junit4]   2> 383066 INFO  (jetty-launcher-310-thread-2) [n:127.0.0.1:46719_solr    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 383067 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[11395C375D5ACE69]-worker) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 383067 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[11395C375D5ACE69]-worker) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 383075 INFO  (zkCallback-324-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5adbe2a1 name:ZooKeeperConnection Watcher:127.0.0.1:52735 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 383075 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[11395C375D5ACE69]-worker) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 383075 INFO  (SUITE-TestSolrCloudWithDelegationTokens-seed#[11395C375D5ACE69]-worker) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 383172 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[11395C375D5ACE69]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenew
   [junit4]   2> 385786 INFO  (qtp1050296361-2049) [n:127.0.0.1:59036_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVcp1_iXigFXTeR8lwEEFPZYhRvkc3NSscVuzctrvdNO5FZ1B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 385786 INFO  (qtp1050296361-2049) [n:127.0.0.1:59036_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVcp1_iXigFXTeR8lwEEFPZYhRvkc3NSscVuzctrvdNO5FZ1B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 386050 INFO  (qtp1909215633-2062) [n:127.0.0.1:46719_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVcp1_iXigFXTeR8lwEEFPZYhRvkc3NSscVuzctrvdNO5FZ1B3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 386050 INFO  (qtp1909215633-2062) [n:127.0.0.1:46719_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVcp1_iXigFXTeR8lwEEFPZYhRvkc3NSscVuzctrvdNO5FZ1B3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 386122 INFO  (qtp1050296361-2048) [n:127.0.0.1:59036_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVcp2AJRigFXTeSGUQIEFB0SSck1SVcUjI01BhDg-Ujha20FB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 386122 INFO  (qtp1050296361-2048) [n:127.0.0.1:59036_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVcp2AJRigFXTeSGUQIEFB0SSck1SVcUjI01BhDg-Ujha20FB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 386174 INFO  (qtp1909215633-2066) [n:127.0.0.1:46719_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVcp2AJRigFXTeSGUQIEFB0SSck1SVcUjI01BhDg-Ujha20FB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 386174 INFO  (qtp1909215633-2066) [n:127.0.0.1:46719_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVcp2AJRigFXTeSGUQIEFB0SSck1SVcUjI01BhDg-Ujha20FB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 386242 INFO  (qtp1050296361-2055) [n:127.0.0.1:59036_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVcp2ALaigFXTeSG2gMEFC2kvJld_tDUXzBWKdDxo4jSVwwGB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 386242 INFO  (qtp1050296361-2055) [n:127.0.0.1:59036_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVcp2ALaigFXTeSG2gMEFC2kvJld_tDUXzBWKdDxo4jSVwwGB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 386274 INFO  (qtp1909215633-2066) [n:127.0.0.1:46719_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVcp2ALaigFXTeSG2gMEFC2kvJld_tDUXzBWKdDxo4jSVwwGB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 386274 INFO  (qtp1909215633-2066) [n:127.0.0.1:46719_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVcp2ALaigFXTeSG2gMEFC2kvJld_tDUXzBWKdDxo4jSVwwGB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 386322 INFO  (qtp1050296361-2055) [n:127.0.0.1:59036_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVcp2AMsigFXTeSHLAQEFJkoiydYtc0rglcHAQRxYTGuLPpGB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 386322 INFO  (qtp1050296361-2055) [n:127.0.0.1:59036_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVcp2AMsigFXTeSHLAQEFJkoiydYtc0rglcHAQRxYTGuLPpGB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 386338 INFO  (qtp1909215633-2066) [n:127.0.0.1:46719_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :list with params action=LIST&delegation=GgADYmFyA2JhcgCKAVcp2AMsigFXTeSHLAQEFJkoiydYtc0rglcHAQRxYTGuLPpGB3NvbHItZHQA&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 386338 INFO  (qtp1909215633-2066) [n:127.0.0.1:46719_solr    ] o.a.s.s.HttpSolrCall [admin] webapp=null path=/admin/collections params={action=LIST&delegation=GgADYmFyA2JhcgCKAVcp2AMsigFXTeSHLAQEFJkoiydYtc0rglcHAQRxYTGuLPpGB3NvbHItZHQA&wt=javabin&version=2} status=0 QTime=0
   [junit4]   2> 386354 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenew-seed#[11395C375D5ACE69]) [    ] o.a.s.SolrTestCaseJ4 ###Ending testDelegationTokenRenew
   [junit4]   2> 386379 INFO  (TEST-TestSolrCloudWithDelegationTokens.testDelegationTokenRenewFail-seed#[11395C375D5ACE69]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testDelegationTokenRenewFail
   [junit4]   2> 386389 WARN  (qtp1050296361-2048) [n:127.0.0.1:59036_solr    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
   [junit4]   2> 386390 WARN  (qtp1050296361-2048) [n:127.0.0.1:59036_solr    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.kerberosplugin.requestcontinues
   [junit4]   2> 386403 WARN  (qtp1909215633-2063) [    ] o.a.h.s.a.s.AuthenticationFilter Authentication exception: org.apache.hadoop.security.AccessControlException: foo tries to renew a token with renewer bar
   [junit4]   2> 386425 WARN  (qtp1909215633-2063) [    ] o.a.s.s.KerberosPlugin Could not find org.apache.solr.security.kerberosplugin.requestcontinues
   [junit4]   2> 386435 WARN  (qtp1050296361-2052) [    ] o.a.h.s.a.s.Authentication

[...truncated too long message...]

b/apacheds-protocol-ldap-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-protocol-shared-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/apacheds-xdbm-partition-2.0.0-M15.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/api-all-1.0.0-M20.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/bcprov-jdk15-1.45.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/cglib-nodep-2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/commons-collections-3.2.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/commons-math3-3.4.1.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/easymock-3.0.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/ehcache-core-2.4.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-common-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-hdfs-2.7.2-tests.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/hadoop-minikdc-2.7.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/htrace-core-3.2.0-incubating.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-annotations-2.5.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-core-asl-1.9.13.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-databind-2.5.4.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jackson-mapper-asl-1.9.13.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jersey-core-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jersey-server-1.9.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/jetty-util-6.1.26.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/mina-core-2.0.0-M5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/mockito-core-1.9.5.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/netty-all-4.0.36.Final.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/objenesis-1.2.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/presto-parser-0.122.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/test-lib/slice-0.10.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/analysis/icu/lucene-analyzers-icu-7.0.0-SNAPSHOT.jar:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/contrib/solr-analysis-extras/classes/java:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/queryparser/classes/test:/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/contrib/analysis-extras/lib/icu4j-56.1.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-launcher.jar:/x1/jenkins/.ant/lib/ivy-2.3.0.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-log4j.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-junit4.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jai.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-javamail.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bsf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-net.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-antlr.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jsch.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-oro.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-commons-logging.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-netrexx.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-testutil.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jdepend.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-bcel.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-xalan2.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-resolver.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-jmf.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-apache-regexp.jar:/home/jenkins/tools/ant/apache-ant-1.8.4/lib/ant-swing.jar:/usr/local/asfpackages/java/jdk1.8.0_102/lib/tools.jar:/x1/jenkins/.ivy2/cache/com.carrotsearch.randomizedtesting/junit4-ant/jars/junit4-ant-2.3.4.jar com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe -eventsfile /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20160914_175010_876.events @/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20160914_175010_876.suites -stdin
   [junit4] ERROR: JVM J2 ended with an exception: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.executeSlave(JUnit4.java:1501)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4.access$000(JUnit4.java:121)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:955)
   [junit4] 	at com.carrotsearch.ant.tasks.junit4.JUnit4$2.call(JUnit4.java:952)
   [junit4] 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4] 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4] 	at java.lang.Thread.run(Thread.java:745)

BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:707: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:59: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build.xml:233: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/common-build.xml:536: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:1443: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/common-build.xml:984: At least one slave process threw an exception, first: Quit event not received from the forked process? This may indicate JVM crash or runner bugs.

Total time: 391 minutes 23 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
ERROR: Failed to archive {checkout/solr/build/solr-core/test/temp/junit4-J2-20160914_175010_876.events=checkout/solr/build/solr-core/test/temp/junit4-J2-20160914_175010_876.events, checkout/solr/build/solr-core/test/temp/junit4-J0-20160914_175010_876.events=checkout/solr/build/solr-core/test/temp/junit4-J0-20160914_175010_876.events, checkout/README.txt=checkout/README.txt, checkout/solr/build/solr-core/test/temp/junit4-J1-20160914_175010_883.events=checkout/solr/build/solr-core/test/temp/junit4-J1-20160914_175010_883.events} due to internal error; falling back to full archiving
java.lang.NullPointerException
	at jsync.protocol.ChangeInputStream.read(ChangeInputStream.java:69)
	at com.cloudbees.jenkins.plugins.jsync.archiver.MD5DigestingInputStream.read(MD5DigestingInputStream.java:39)
	at com.google.common.io.LimitInputStream.read(LimitInputStream.java:79)
	at java.io.FilterInputStream.read(FilterInputStream.java:107)
	at com.google.common.io.ByteStreams.copy(ByteStreams.java:193)
	at jsync.protocol.FileSequenceReader.read(FileSequenceReader.java:35)
	at com.cloudbees.jenkins.plugins.jsync.archiver.JSyncArtifactManager.remoteSync(JSyncArtifactManager.java:158)
	at com.cloudbees.jenkins.plugins.jsync.archiver.JSyncArtifactManager.archive(JSyncArtifactManager.java:76)
	at hudson.tasks.ArtifactArchiver.perform(ArtifactArchiver.java:238)
	at hudson.tasks.BuildStepCompatibilityLayer.perform(BuildStepCompatibilityLayer.java:78)
	at hudson.tasks.BuildStepMonitor$1.perform(BuildStepMonitor.java:20)
	at hudson.model.AbstractBuild$AbstractBuildExecution.perform(AbstractBuild.java:779)
	at hudson.model.AbstractBuild$AbstractBuildExecution.performAllBuildSteps(AbstractBuild.java:720)
	at hudson.model.Build$BuildExecution.post2(Build.java:185)
	at hudson.model.AbstractBuild$AbstractBuildExecution.post(AbstractBuild.java:665)
	at hudson.model.Run.execute(Run.java:1766)
	at hudson.model.FreeStyleBuild.run(FreeStyleBuild.java:43)
	at hudson.model.ResourceController.execute(ResourceController.java:98)
	at hudson.model.Executor.run(Executor.java:410)
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1115 - Failure

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1115/

4 tests failed.
FAILED:  org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:38503/ff_/i

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:38503/ff_/i
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:477)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  org.apache.solr.handler.component.DistributedFacetPivotLongTailTest.test

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded
	at __randomizedtesting.SeedInfo.seed([B9E25108DCDC63C8:31B66ED272200E30]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:347)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:152)
	at org.apache.solr.handler.component.DistributedFacetPivotLongTailTest.test(DistributedFacetPivotLongTailTest.java:71)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
	at __randomizedtesting.SeedInfo.seed([B9E25108DCDC63C8]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.GeneratedMethodAccessor99.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest:     1) Thread[id=197403, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=197393, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=197423, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=197408, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=197418, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=197398, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=197378, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=197383, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=197413, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=197388, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 
   1) Thread[id=197403, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=197393, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=197423, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=197408, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=197418, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=197398, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=197378, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=197383, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=197413, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=197388, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([B9E25108DCDC63C8]:0)




Build Log:
[...truncated 12778 lines...]
   [junit4] Suite: org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_B9E25108DCDC63C8-001/init-core-data-001
   [junit4]   2> 5777874 INFO  (SUITE-ConcurrentDeleteAndCreateCollectionTest-seed#[B9E25108DCDC63C8]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 5777877 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testConcurrentCreateAndDeleteDoesNotFail
   [junit4]   2> 5777877 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 5777886 INFO  (Thread-181698) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 5777887 INFO  (Thread-181698) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 5778051 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.ZkTestServer start zk server on port:43889
   [junit4]   2> 5778051 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778053 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778073 INFO  (zkCallback-3735-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@feb53a6 name:ZooKeeperConnection Watcher:127.0.0.1:43889 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778073 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778073 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778074 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr/solr.xml
   [junit4]   2> 5778079 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr/clusterprops.json
   [junit4]   2> 5778086 INFO  (jetty-launcher-3734-thread-1) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 5778093 INFO  (jetty-launcher-3734-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@75838d63{/solr,null,AVAILABLE}
   [junit4]   2> 5778095 INFO  (jetty-launcher-3734-thread-1) [    ] o.e.j.s.ServerConnector Started ServerConnector@6f98c6eb{SSL,[ssl, http/1.1]}{127.0.0.1:60459}
   [junit4]   2> 5778095 INFO  (jetty-launcher-3734-thread-1) [    ] o.e.j.s.Server Started @5785706ms
   [junit4]   2> 5778099 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=60459}
   [junit4]   2> 5778099 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 5778099 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_B9E25108DCDC63C8-001/tempDir-001/node1'
   [junit4]   2> 5778099 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 5778100 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 5778100 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778110 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778110 INFO  (zkCallback-3736-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@753ea486 name:ZooKeeperConnection Watcher:127.0.0.1:43889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778110 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778110 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778110 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 5778123 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_B9E25108DCDC63C8-001/tempDir-001/node1/.
   [junit4]   2> 5778123 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.CoreContainer New CoreContainer 1450366150
   [junit4]   2> 5778123 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_B9E25108DCDC63C8-001/tempDir-001/node1]
   [junit4]   2> 5778123 WARN  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.CoreContainer Couldn't add files from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_B9E25108DCDC63C8-001/tempDir-001/node1/lib to classpath: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_B9E25108DCDC63C8-001/tempDir-001/node1/lib
   [junit4]   2> 5778124 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : https,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,
   [junit4]   2> 5778126 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.u.UpdateShardHandler Created UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&maxConnections=100000&maxConnectionsPerHost=100000
   [junit4]   2> 5778126 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 5778126 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 5778126 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:43889/solr
   [junit4]   2> 5778126 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 5778126 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778127 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778143 INFO  (zkCallback-3739-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@13d9a076 name:ZooKeeperConnection Watcher:127.0.0.1:43889 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778143 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778143 INFO  (jetty-launcher-3734-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778146 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$12/1518713987@4a0f334
   [junit4]   2> 5778155 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778158 INFO  (zkCallback-3740-thread-1-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@60ab6f6d name:ZooKeeperConnection Watcher:127.0.0.1:43889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778158 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778159 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
   [junit4]   2> 5778160 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
   [junit4]   2> 5778162 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
   [junit4]   2> 5778164 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
   [junit4]   2> 5778172 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
   [junit4]   2> 5778173 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 5778174 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 5778174 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 5778175 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 5778179 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 5778184 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 5778185 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ZkStateReader Loaded cluster properties: {urlScheme=https}
   [junit4]   2> 5778185 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 5778187 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
   [junit4]   2> 5778197 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
   [junit4]   2> 5778198 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 5778198 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/96581753093685251-127.0.0.1:60459_solr-n_0000000000
   [junit4]   2> 5778198 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:60459_solr
   [junit4]   2> 5778198 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
   [junit4]   2> 5778208 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.Overseer Overseer (id=96581753093685251-127.0.0.1:60459_solr-n_0000000000) starting
   [junit4]   2> 5778209 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
   [junit4]   2> 5778222 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 5778223 INFO  (OverseerCollectionConfigSetProcessor-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
   [junit4]   2> 5778231 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60459_solr
   [junit4]   2> 5778231 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:60459_solr
   [junit4]   2> 5778231 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.Overseer Starting to work on the main queue
   [junit4]   2> 5778247 INFO  (zkCallback-3740-thread-1-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 5778248 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 5778275 INFO  (zkCallback-3740-thread-1-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (1)
   [junit4]   2> 5778280 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 5778280 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 5778281 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_B9E25108DCDC63C8-001/tempDir-001/node1/.
   [junit4]   2> 5778281 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions
   [junit4]   2> 5778295 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.s.SolrDispatchFilter user.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2
   [junit4]   2> 5778295 INFO  (jetty-launcher-3734-thread-1) [n:127.0.0.1:60459_solr    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 5778299 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778301 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778301 INFO  (zkCallback-3742-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4f6830a0 name:ZooKeeperConnection Watcher:127.0.0.1:43889 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778301 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778301 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778316 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778347 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778349 INFO  (zkCallback-3744-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5feb1f6e name:ZooKeeperConnection Watcher:127.0.0.1:43889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778349 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778350 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778350 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection0/schema.xml
   [junit4]   2> 5778352 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection0/solrconfig.xml
   [junit4]   2> 5778367 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778376 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778377 INFO  (zkCallback-3745-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@60b9fb04 name:ZooKeeperConnection Watcher:127.0.0.1:43889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778377 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778377 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778378 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection1/schema.xml
   [junit4]   2> 5778387 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection1/solrconfig.xml
   [junit4]   2> 5778399 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778400 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778410 INFO  (zkCallback-3746-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@24ba16e2 name:ZooKeeperConnection Watcher:127.0.0.1:43889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778410 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778410 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778411 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection2/schema.xml
   [junit4]   2> 5778463 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection2/solrconfig.xml
   [junit4]   2> 5778474 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778479 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778499 INFO  (zkCallback-3747-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@138f6e63 name:ZooKeeperConnection Watcher:127.0.0.1:43889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778499 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778499 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778500 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection3/schema.xml
   [junit4]   2> 5778505 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection3/solrconfig.xml
   [junit4]   2> 5778518 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778523 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778527 INFO  (zkCallback-3748-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@7cb113fc name:ZooKeeperConnection Watcher:127.0.0.1:43889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778527 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778528 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778528 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection4/schema.xml
   [junit4]   2> 5778530 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection4/solrconfig.xml
   [junit4]   2> 5778537 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778547 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778549 INFO  (zkCallback-3749-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@42690972 name:ZooKeeperConnection Watcher:127.0.0.1:43889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778549 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778549 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778550 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection5/schema.xml
   [junit4]   2> 5778557 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection5/solrconfig.xml
   [junit4]   2> 5778600 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778603 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778615 INFO  (zkCallback-3750-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5935097b name:ZooKeeperConnection Watcher:127.0.0.1:43889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778615 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778616 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778616 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection6/schema.xml
   [junit4]   2> 5778659 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection6/solrconfig.xml
   [junit4]   2> 5778687 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778701 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778715 INFO  (zkCallback-3751-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5e246904 name:ZooKeeperConnection Watcher:127.0.0.1:43889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778715 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778715 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778716 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection7/schema.xml
   [junit4]   2> 5778717 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection7/solrconfig.xml
   [junit4]   2> 5778765 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778795 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778796 INFO  (zkCallback-3752-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@64985ce9 name:ZooKeeperConnection Watcher:127.0.0.1:43889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778796 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778796 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778797 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection8/schema.xml
   [junit4]   2> 5778798 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection8/solrconfig.xml
   [junit4]   2> 5778807 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 5778827 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 5778835 INFO  (zkCallback-3753-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3ff66111 name:ZooKeeperConnection Watcher:127.0.0.1:43889/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 5778835 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 5778835 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 5778836 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection9/schema.xml
   [junit4]   2> 5778838 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteDoesNotFail-seed#[B9E25108DCDC63C8]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/collection9/solrconfig.xml
   [junit4]   2> 5779276 INFO  (qtp1703007059-197353) [n:127.0.0.1:60459_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=collection0&name=collection0&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 5779278 INFO  (OverseerCollectionConfigSetProcessor-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2>   "name":"collection0",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection0",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779281 INFO  (qtp1703007059-197346) [n:127.0.0.1:60459_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=collection1&name=collection1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 5779329 INFO  (qtp1703007059-197347) [n:127.0.0.1:60459_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=collection4&name=collection4&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 5779340 INFO  (qtp1703007059-197352) [n:127.0.0.1:60459_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=collection8&name=collection8&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 5779343 INFO  (qtp1703007059-197428) [n:127.0.0.1:60459_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=collection9&name=collection9&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 5779345 INFO  (qtp1703007059-197351) [n:127.0.0.1:60459_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=collection7&name=collection7&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 5779347 INFO  (OverseerCollectionConfigSetProcessor-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000006 message:{
   [junit4]   2>   "name":"collection7",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection7",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779348 INFO  (OverseerThreadFactory-8116-thread-1-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"collection0",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection0",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779348 INFO  (OverseerThreadFactory-8116-thread-1-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection0
   [junit4]   2> 5779352 INFO  (qtp1703007059-197349) [n:127.0.0.1:60459_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=collection3&name=collection3&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 5779391 INFO  (OverseerThreadFactory-8116-thread-2-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"collection7",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection7",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779391 INFO  (OverseerThreadFactory-8116-thread-2-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection7
   [junit4]   2> 5779411 INFO  (qtp1703007059-197427) [n:127.0.0.1:60459_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=collection2&name=collection2&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 5779412 INFO  (qtp1703007059-197431) [n:127.0.0.1:60459_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=collection5&name=collection5&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 5779428 INFO  (OverseerThreadFactory-8116-thread-1-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection0 
   [junit4]   2> 5779428 INFO  (OverseerThreadFactory-8116-thread-2-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection7 
   [junit4]   2> 5779430 INFO  (OverseerThreadFactory-8116-thread-1-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection0
   [junit4]   2> 5779431 INFO  (zkCallback-3740-thread-1-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 5779431 INFO  (OverseerCollectionConfigSetProcessor-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000003 message:{
   [junit4]   2>   "name":"collection4",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection4",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779434 INFO  (OverseerThreadFactory-8116-thread-2-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection7
   [junit4]   2> 5779448 INFO  (qtp1703007059-197350) [n:127.0.0.1:60459_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=collection6&name=collection6&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 5779547 INFO  (OverseerCollectionConfigSetProcessor-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000004 message:{
   [junit4]   2>   "name":"collection8",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection8",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779547 INFO  (OverseerThreadFactory-8116-thread-3-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"collection4",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection4",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779547 INFO  (OverseerCollectionConfigSetProcessor-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000005 message:{
   [junit4]   2>   "name":"collection9",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection9",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779548 INFO  (OverseerThreadFactory-8116-thread-3-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection4
   [junit4]   2> 5779548 INFO  (OverseerThreadFactory-8116-thread-4-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"collection8",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection8",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779548 INFO  (OverseerThreadFactory-8116-thread-4-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection8
   [junit4]   2> 5779549 INFO  (zkCallback-3740-thread-2-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 5779549 INFO  (OverseerThreadFactory-8116-thread-5-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"collection9",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection9",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779549 INFO  (OverseerThreadFactory-8116-thread-5-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection9
   [junit4]   2> 5779552 INFO  (OverseerThreadFactory-8116-thread-3-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection4 
   [junit4]   2> 5779552 INFO  (OverseerThreadFactory-8116-thread-4-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection8 
   [junit4]   2> 5779553 INFO  (OverseerThreadFactory-8116-thread-5-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection9 
   [junit4]   2> 5779567 INFO  (OverseerThreadFactory-8116-thread-4-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection8
   [junit4]   2> 5779567 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
   [junit4]   2>   "name":"collection0",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection0",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 5779567 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection0
   [junit4]   2> 5779567 INFO  (OverseerThreadFactory-8116-thread-3-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection4
   [junit4]   2> 5779567 INFO  (OverseerThreadFactory-8116-thread-5-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection9
   [junit4]   2> 5779573 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection0/state.json
   [junit4]   2> 5779574 INFO  (zkCallback-3740-thread-1-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 5779587 INFO  (OverseerCollectionConfigSetProcessor-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000002 message:{
   [junit4]   2>   "name":"collection1",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection1",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779590 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.Overseer processMessage: queueSize: 3, message = {
   [junit4]   2>   "name":"collection7",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection7",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 5779590 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection7
   [junit4]   2> 5779598 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
   [junit4]   2>   "name":"collection4",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection4",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 5779598 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection4
   [junit4]   2> 5779604 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection7/state.json
   [junit4]   2> 5779606 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "name":"collection8",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection8",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 5779606 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection8
   [junit4]   2> 5779607 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection4/state.json
   [junit4]   2> 5779612 INFO  (OverseerCollectionConfigSetProcessor-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000010 message:{
   [junit4]   2>   "name":"collection3",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection3",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779614 INFO  (OverseerThreadFactory-8116-thread-6-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"collection1",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection1",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779615 INFO  (OverseerThreadFactory-8116-thread-6-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 5779627 INFO  (OverseerThreadFactory-8116-thread-6-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection1 
   [junit4]   2> 5779628 INFO  (OverseerThreadFactory-8116-thread-6-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1
   [junit4]   2> 5779628 INFO  (zkCallback-3740-thread-2-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 5779629 INFO  (zkCallback-3740-thread-2-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 5779636 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "name":"collection1",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection1",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 5779636 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection1
   [junit4]   2> 5779636 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection8/state.json
   [junit4]   2> 5779640 INFO  (OverseerThreadFactory-8116-thread-1-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Creating SolrCores for new collection collection0, shardNames [shard1] , replicationFactor : 1
   [junit4]   2> 5779640 INFO  (OverseerThreadFactory-8116-thread-1-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Creating core collection0_shard1_replica1 as part of shard shard1 of collection collection0 on 127.0.0.1:60459_solr
   [junit4]   2> 5779643 INFO  (OverseerThreadFactory-8116-thread-2-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Creating SolrCores for new collection collection7, shardNames [shard1] , replicationFactor : 1
   [junit4]   2> 5779643 INFO  (OverseerThreadFactory-8116-thread-2-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Creating core collection7_shard1_replica1 as part of shard shard1 of collection collection7 on 127.0.0.1:60459_solr
   [junit4]   2> 5779667 INFO  (OverseerCollectionConfigSetProcessor-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000011 message:{
   [junit4]   2>   "name":"collection2",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection2",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779667 INFO  (OverseerThreadFactory-8116-thread-7-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"collection3",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection3",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779667 INFO  (OverseerThreadFactory-8116-thread-7-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection3
   [junit4]   2> 5779668 INFO  (OverseerThreadFactory-8116-thread-7-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection3 
   [junit4]   2> 5779668 INFO  (OverseerThreadFactory-8116-thread-7-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection3
   [junit4]   2> 5779670 INFO  (OverseerCollectionConfigSetProcessor-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000012 message:{
   [junit4]   2>   "name":"collection5",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection5",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779670 INFO  (OverseerThreadFactory-8116-thread-8-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"collection2",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection2",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779670 INFO  (OverseerThreadFactory-8116-thread-8-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection2
   [junit4]   2> 5779673 INFO  (OverseerThreadFactory-8116-thread-8-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection2 
   [junit4]   2> 5779673 INFO  (OverseerThreadFactory-8116-thread-8-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection2
   [junit4]   2> 5779676 INFO  (OverseerCollectionConfigSetProcessor-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000018 message:{
   [junit4]   2>   "name":"collection6",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection6",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779676 INFO  (OverseerThreadFactory-8116-thread-9-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"collection5",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection5",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779676 INFO  (OverseerThreadFactory-8116-thread-9-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection5
   [junit4]   2> 5779686 INFO  (OverseerThreadFactory-8116-thread-9-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection5 
   [junit4]   2> 5779687 INFO  (OverseerThreadFactory-8116-thread-9-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection5
   [junit4]   2> 5779695 INFO  (OverseerThreadFactory-8116-thread-10-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"collection6",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection6",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 5779695 INFO  (OverseerThreadFactory-8116-thread-10-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection6
   [junit4]   2> 5779699 INFO  (OverseerThreadFactory-8116-thread-3-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Creating SolrCores for new collection collection4, shardNames [shard1] , replicationFactor : 1
   [junit4]   2> 5779699 INFO  (OverseerThreadFactory-8116-thread-3-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Creating core collection4_shard1_replica1 as part of shard shard1 of collection collection4 on 127.0.0.1:60459_solr
   [junit4]   2> 5779699 INFO  (OverseerThreadFactory-8116-thread-4-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Creating SolrCores for new collection collection8, shardNames [shard1] , replicationFactor : 1
   [junit4]   2> 5779699 INFO  (OverseerThreadFactory-8116-thread-4-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Creating core collection8_shard1_replica1 as part of shard shard1 of collection collection8 on 127.0.0.1:60459_solr
   [junit4]   2> 5779710 INFO  (OverseerThreadFactory-8116-thread-10-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection6 
   [junit4]   2> 5779710 INFO  (OverseerThreadFactory-8116-thread-10-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection6
   [junit4]   2> 5779739 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection1/state.json
   [junit4]   2> 5779741 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.Overseer processMessage: queueSize: 5, message = {
   [junit4]   2>   "name":"collection9",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection9",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 5779741 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection9
   [junit4]   2> 5779743 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.Overseer processMessage: queueSize: 0, message = {
   [junit4]   2>   "name":"collection3",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection3",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 5779743 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection3
   [junit4]   2> 5779751 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection9/state.json
   [junit4]   2> 5779754 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.Overseer processMessage: queueSize: 0, message = {
   [junit4]   2>   "name":"collection2",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection2",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 5779754 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection2
   [junit4]   2> 5779754 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection3/state.json
   [junit4]   2> 5779756 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.Overseer processMessage: queueSize: 0, message = {
   [junit4]   2>   "name":"collection5",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection5",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 5779756 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection5
   [junit4]   2> 5779757 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection2/state.json
   [junit4]   2> 5779765 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.Overseer processMessage: queueSize: 0, message = {
   [junit4]   2>   "name":"collection6",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"collection6",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 5779765 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection6
   [junit4]   2> 5779765 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection5/state.json
   [junit4]   2> 5779838 INFO  (OverseerThreadFactory-8116-thread-6-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Creating SolrCores for new collection collection1, shardNames [shard1] , replicationFactor : 1
   [junit4]   2> 5779838 INFO  (OverseerThreadFactory-8116-thread-6-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Creating core collection1_shard1_replica1 as part of shard shard1 of collection collection1 on 127.0.0.1:60459_solr
   [junit4]   2> 5779856 INFO  (OverseerThreadFactory-8116-thread-5-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Creating SolrCores for new collection collection9, shardNames [shard1] , replicationFactor : 1
   [junit4]   2> 5779856 INFO  (OverseerThreadFactory-8116-thread-5-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.CreateCollectionCmd Creating core collection9_shard1_replica1 as part of shard shard1 of collection collection9 on 127.0.0.1:60459_solr
   [junit4]   2> 5779871 INFO  (OverseerStateUpdate-96581753093685251-127.0.0.1:60459_solr-n_0000000000) [n:127.0.0.1:60459_solr    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection6/state.json
   [junit4]   2> 5780157 INFO  (zkCallback-3740-thread-1-processing-n:127.0.0.1:60459_solr) [n:127.0.0.1:60459_solr    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 5780205 INFO  (qtp1703007059-197433) [n:127.0.0.1:60459_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=collection0&name=collection0_shard1_replica1&action=CREATE&numShards=1&collection=collection0&shard=shard1&wt=javabin&version=2
   [junit4]   2> 5780206 INFO  (qtp1703007059-197435) [n:127.0.0.1:60459_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=collection9&name=collection9_shard1_replica1&action=CREATE&numShards=1&collection=collection9&shard=shard1&wt=javabin&version=2
   [junit4]   2> 5780206 INFO  (qtp1703007059-197433) [n:127.0.0.1:60459_solr    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection0_shard1_replica1, collection.configName=collection0, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.json, collection=collection0, numShards=1, dataDir=data/, shard=shard1}
   [junit4]   2> 5780206 INFO  (qtp1703007059-197435) [n:127.0.0.1:60459_solr    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection9_shard1_replica1, collection.configName=collection9, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.json, collection=collection9, numShards=1, dataDir=data/, shard=shard1}
   [junit4]   2> 5780206 INFO  (qtp1703007059-197426) [n:127.0.0.1:60459_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=collection7&name=collection7_shard1_replica1&action=CREATE&numShards=1&collection=collection7&shard=shard1&wt=javabin&version=2
   [junit4]   2> 5780206 INFO  (qtp1703007059-197426) [n:127.0.0.1:60459_solr    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection7_shard1_replica1, collection.configName=collection7, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.json, collection=collection7, numShards=1, dataDir=data/, shard=shard1}
   [junit4]   2> 5780207 INFO  (qtp1703007059-197426) [n:127.0.0.1:60459_solr c:collection7 s:shard1  x:collection7_shard1_replica1] o.a.s.c.ZkController publishing state=down
   [junit4]   2> 5780208 INFO  (qtp1703007059-197426) [n:127.0.0.1:60459_solr c:collection7 s:shard1  x:collection7_shard1_replica1] o.a.s.c.ZkController Registering watch for collection collection7
   [junit4]  

[...truncated too long message...]

ection1/data
   [junit4]   2> 7480269 INFO  (coreCloseExecutor-11532-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory looking to close /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedFacetPivotLongTailTest_B9E25108DCDC63C8-001/tempDir-001/shard2/cores/collection1/data/index [CachedDir<<refCount=0;path=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedFacetPivotLongTailTest_B9E25108DCDC63C8-001/tempDir-001/shard2/cores/collection1/data/index;done=false>>]
   [junit4]   2> 7480269 INFO  (coreCloseExecutor-11532-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory Closing directory, CoreContainer#isShutdown=true
   [junit4]   2> 7480269 INFO  (coreCloseExecutor-11532-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory Closing directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedFacetPivotLongTailTest_B9E25108DCDC63C8-001/tempDir-001/shard2/cores/collection1/data/index
   [junit4]   2> 7480271 INFO  (TEST-DistributedFacetPivotLongTailTest.test-seed#[B9E25108DCDC63C8]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@55954cfb{/,null,UNAVAILABLE}
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=DistributedFacetPivotLongTailTest -Dtests.method=test -Dtests.seed=B9E25108DCDC63C8 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=is -Dtests.timezone=Mexico/BajaSur -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR    116s J0 | DistributedFacetPivotLongTailTest.test <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([B9E25108DCDC63C8:31B66ED272200E30]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.createMethod(HttpSolrClient.java:347)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:259)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:152)
   [junit4]    > 	at org.apache.solr.handler.component.DistributedFacetPivotLongTailTest.test(DistributedFacetPivotLongTailTest.java:71)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
   [junit4]   2> 7480332 INFO  (SUITE-DistributedFacetPivotLongTailTest-seed#[B9E25108DCDC63C8]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedFacetPivotLongTailTest_B9E25108DCDC63C8-001
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=ClassicSimilarity, locale=is, timezone=Mexico/BajaSur
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=92830728,total=477626368
   [junit4]   2> NOTE: All tests run in this JVM: [TestRealTimeGet, TestFieldCacheVsDocValues, ExplicitHLLTest, PreAnalyzedUpdateProcessorTest, TestCloudPseudoReturnFields, TestScoreJoinQPNoScore, MBeansHandlerTest, SolrCloudExampleTest, ZkNodePropsTest, TestNoOpRegenerator, TestFoldingMultitermQuery, TestUseDocValuesAsStored, TestJsonFacets, TestRandomDVFaceting, TestCustomSort, ResponseHeaderTest, TestIBSimilarityFactory, HdfsNNFailoverTest, UUIDFieldTest, TestRawTransformer, TestConfigSetProperties, TestCopyFieldCollectionResource, RequiredFieldsTest, TestPKIAuthenticationPlugin, TestHdfsUpdateLog, ReturnFieldsTest, RegexBoostProcessorTest, TestStressRecovery, AlternateDirectoryTest, TestNumericTerms32, TestPseudoReturnFields, BlockJoinFacetRandomTest, TestFaceting, WrapperMergePolicyFactoryTest, LeaderInitiatedRecoveryOnShardRestartTest, FileBasedSpellCheckerTest, TestComponentsName, RulesTest, HdfsChaosMonkeyNothingIsSafeTest, TestLocalFSCloudBackupRestore, DateRangeFieldTest, OverseerTaskQueueTest, ScriptEngineTest, TestNRTOpen, TestReplicationHandler, NumericFieldsTest, TestTrieFacet, DistributedSuggestComponentTest, TestRecoveryHdfs, TestJoin, TestRandomFlRTGCloud, TestSolr4Spatial, SynonymTokenizerTest, TestExpandComponent, TestMacroExpander, TestSolrCloudWithDelegationTokens, TestRandomCollapseQParserPlugin, ChaosMonkeyNothingIsSafeTest, CollectionStateFormat2Test, StandardRequestHandlerTest, CleanupOldIndexTest, TestHighlightDedupGrouping, CursorPagingTest, TestValueSourceCache, TestGeoJSONResponseWriter, TestTolerantUpdateProcessorRandomCloud, ConnectionReuseTest, DocumentAnalysisRequestHandlerTest, HLLUtilTest, TestDynamicFieldResource, DataDrivenBlockJoinTest, TestBackupRepositoryFactory, TestRestManager, QueryEqualityTest, DistributedFacetPivotWhiteBoxTest, BasicAuthIntegrationTest, CoreAdminRequestStatusTest, PeerSyncTest, ResourceLoaderTest, TestHdfsCloudBackupRestore, TestFastWriter, OutOfBoxZkACLAndCredentialsProvidersTest, NotRequiredUniqueKeyTest, TestReqParamsAPI, TestPHPSerializedResponseWriter, TestSchemaResource, TestReplicaProperties, TestHighFrequencyDictionaryFactory, AnalysisAfterCoreReloadTest, TestFunctionQuery, XsltUpdateRequestHandlerTest, ClusterStateTest, DefaultValueUpdateProcessorTest, TestStressCloudBlindAtomicUpdates, TestSolrConfigHandler, UUIDUpdateProcessorFallbackTest, AsyncCallRequestStatusResponseTest, CdcrRequestHandlerTest, ZkStateWriterTest, TestSolrDeletionPolicy2, TestDistribDocBasedVersion, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, UnloadDistributedZkTest, CollectionsAPIDistributedZkTest, FullSolrCloudDistribCmdsTest, ZkSolrClientTest, TestDistributedGrouping, TestHashPartitioner, ZkControllerTest, TestStressReorder, HardAutoCommitTest, TestRangeQuery, SolrCoreTest, SpellCheckComponentTest, TestGroupingSearch, HighlighterTest, ShowFileRequestHandlerTest, CurrencyFieldOpenExchangeTest, DistributedQueryElevationComponentTest, SimplePostToolTest, TestCoreDiscovery, CoreAdminHandlerTest, TestStressLucene, SuggesterWFSTTest, PolyFieldTest, TestAtomicUpdateErrorCases, SortByFunctionTest, DocValuesMultiTest, DebugComponentTest, IndexBasedSpellCheckerTest, DisMaxRequestHandlerTest, TestWriterPerf, XmlUpdateRequestHandlerTest, PathHierarchyTokenizerFactoryTest, TestIndexingPerformance, MoreLikeThisHandlerTest, FastVectorHighlighterTest, TestJmxIntegration, UniqFieldsUpdateProcessorFactoryTest, TestPartialUpdateDeduplication, SearchHandlerTest, ExternalFileFieldSortTest, TestSolrCoreProperties, SpellPossibilityIteratorTest, TestCodecSupport, TestBM25SimilarityFactory, TestFastOutputStream, TestFastLRUCache, TestSystemIdResolver, DateFieldTest, SpellingQueryConverterTest, DOMUtilTest, SliceStateTest, SystemInfoHandlerTest, CircularListTest, DistributedIntervalFacetingTest, CdcrVersionReplicationTest, CollectionTooManyReplicasTest, CollectionsAPISolrJTest, ConfigSetsAPITest, ConnectionManagerTest, CustomCollectionTest, DistribDocExpirationUpdateProcessorTest, HttpPartitionTest, SaslZkACLProviderTest, TestConfigSetsAPIExclusivity, TestDownShardTolerantSearch, TestLeaderElectionZkExpiry, TestLeaderInitiatedRecoveryThread, TestSizeLimitedDistributedMap, VMParamsZkACLAndCredentialsProvidersTest, HdfsChaosMonkeySafeLeaderTest, HdfsCollectionsAPIDistributedZkTest, HdfsUnloadDistributedZkTest, HdfsWriteToMultipleCollectionsTest, StressHdfsTest, CachingDirectoryFactoryTest, TestImplicitCoreProperties, TestReloadAndDeleteDocs, TestShardHandlerFactory, TestSolrDynamicMBean, TestSolrXml, TestSolrCoreSnapshots, RequestLoggingTest, TestBlobHandler, TestReplicationHandlerBackup, TestSolrConfigHandlerConcurrent, DistributedFacetExistsSmallTest, DistributedFacetPivotLongTailTest]
   [junit4] Completed [554/631 (3!)] on J0 in 116.56s, 1 test, 1 error <<< FAILURES!

[...truncated 222 lines...]
   [junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20160912_211331_976.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid12824.hprof ...
   [junit4] Heap dump file created [494369350 bytes in 7.364 secs]
   [junit4] <<< JVM J0: EOF ----

[...truncated 11045 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:715: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid12824.hprof

Total time: 446 minutes 3 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1114 - Still unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1114/

9 tests failed.
FAILED:  org.apache.lucene.analysis.core.TestRandomChains.testRandomChainsWithLargeStrings

Error Message:


Stack Trace:
java.util.NoSuchElementException
	at __randomizedtesting.SeedInfo.seed([B9ED0ECBBD7CADDD:D3B6B1DAE4328D2E]:0)
	at java.util.TreeMap.key(TreeMap.java:1327)
	at java.util.TreeMap.lastKey(TreeMap.java:297)
	at java.util.TreeSet.last(TreeSet.java:401)
	at org.apache.lucene.analysis.minhash.MinHashFilter$FixedSizeTreeSet.add(MinHashFilter.java:325)
	at org.apache.lucene.analysis.minhash.MinHashFilter.incrementToken(MinHashFilter.java:159)
	at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:67)
	at org.apache.lucene.analysis.util.ElisionFilter.incrementToken(ElisionFilter.java:52)
	at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:67)
	at org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter.incrementToken(ASCIIFoldingFilter.java:102)
	at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:67)
	at org.apache.lucene.analysis.FilteringTokenFilter.incrementToken(FilteringTokenFilter.java:51)
	at org.apache.lucene.analysis.ngram.NGramTokenFilter.incrementToken(NGramTokenFilter.java:98)
	at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:67)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkResetException(BaseTokenStreamTestCase.java:405)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:510)
	at org.apache.lucene.analysis.core.TestRandomChains.testRandomChainsWithLargeStrings(TestRandomChains.java:959)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationStartStop

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard2

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard2
	at __randomizedtesting.SeedInfo.seed([69C7C3B4D964326A:EA041D27CDBCB9E3]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationStartStop(CdcrReplicationDistributedZkTest.java:168)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.core.HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig

Error Message:
The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~134.217728 MB.

Stack Trace:
java.lang.RuntimeException: The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args) or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap, your java heap size might not be large enough. Failed allocating ~134.217728 MB.
	at __randomizedtesting.SeedInfo.seed([69C7C3B4D964326A:9E680A9F24EDD841]:0)
	at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:304)
	at org.apache.solr.core.HdfsDirectoryFactory.getBlockDirectoryCache(HdfsDirectoryFactory.java:286)
	at org.apache.solr.core.HdfsDirectoryFactory.create(HdfsDirectoryFactory.java:220)
	at org.apache.solr.core.HdfsDirectoryFactoryTest.testInitArgsOrSysPropConfig(HdfsDirectoryFactoryTest.java:108)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.OutOfMemoryError: Direct buffer memory
	at java.nio.Bits.reserveMemory(Bits.java:693)
	at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123)
	at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)
	at org.apache.solr.store.blockcache.BlockCache.<init>(BlockCache.java:63)
	at org.apache.solr.core.HdfsDirectoryFactory.createBlockCache(HdfsDirectoryFactory.java:302)
	... 42 more


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard2

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard2
	at __randomizedtesting.SeedInfo.seed([69C7C3B4D964326A:35DA24351744865C]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart(CdcrReplicationDistributedZkTest.java:236)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.handler.TestSolrConfigHandlerCloud

Error Message:
ObjectTracker found 8 object(s) that were not released!!! [MDCAwareThreadPoolExecutor, MockDirectoryWrapper, SolrCore, MockDirectoryWrapper, MDCAwareThreadPoolExecutor, SolrCore, MDCAwareThreadPoolExecutor, MockDirectoryWrapper]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 8 object(s) that were not released!!! [MDCAwareThreadPoolExecutor, MockDirectoryWrapper, SolrCore, MockDirectoryWrapper, MDCAwareThreadPoolExecutor, SolrCore, MDCAwareThreadPoolExecutor, MockDirectoryWrapper]
	at __randomizedtesting.SeedInfo.seed([69C7C3B4D964326A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=17331, name=testExecutor-3503-thread-6, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=17331, name=testExecutor-3503-thread-6, state=RUNNABLE, group=TGRP-HdfsUnloadDistributedZkTest]
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:49991: Error CREATEing SolrCore 'multiunload5': Unable to create core [multiunload5] Caused by: Direct buffer memory
	at __randomizedtesting.SeedInfo.seed([69C7C3B4D964326A]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:608)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.BasicDistributedZkTest.lambda$createCores$0(BasicDistributedZkTest.java:582)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:45599

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:45599
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:516)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
	at __randomizedtesting.SeedInfo.seed([69C7C3B4D964326A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.GeneratedMethodAccessor18.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest:     1) Thread[id=3578, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=3583, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=3598, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=3588, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=3593, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=3558, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=3563, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=3603, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=3573, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=3568, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 
   1) Thread[id=3578, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=3583, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=3598, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=3588, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=3593, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=3558, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=3563, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=3603, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=3573, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=3568, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([69C7C3B4D964326A]:0)




Build Log:
[...truncated 3051 lines...]
   [junit4] Suite: org.apache.lucene.analysis.core.TestRandomChains
   [junit4]   2> Exception from random analyzer: 
   [junit4]   2> charfilters=
   [junit4]   2> tokenizer=
   [junit4]   2>   org.apache.lucene.analysis.core.LetterTokenizer()
   [junit4]   2> filters=
   [junit4]   2>   org.apache.lucene.analysis.minhash.MinHashFilter(ValidatingTokenFilter@1255a487 term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word, 3, 48, -35, true)
   [junit4]   2>   org.apache.lucene.analysis.util.ElisionFilter(ValidatingTokenFilter@7a0b7f3c term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word, [klglj])
   [junit4]   2>   org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter(ValidatingTokenFilter@8ed0d9d term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word, true)
   [junit4]   2>   org.apache.lucene.analysis.ngram.NGramTokenFilter(ValidatingTokenFilter@43c891a term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word)
   [junit4]   2> offsetsAreCorrect=true
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestRandomChains -Dtests.method=testRandomChainsWithLargeStrings -Dtests.seed=B9ED0ECBBD7CADDD -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=ru -Dtests.timezone=Africa/Monrovia -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   25.1s J0 | TestRandomChains.testRandomChainsWithLargeStrings <<<
   [junit4]    > Throwable #1: java.util.NoSuchElementException
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([B9ED0ECBBD7CADDD:D3B6B1DAE4328D2E]:0)
   [junit4]    > 	at java.util.TreeMap.key(TreeMap.java:1327)
   [junit4]    > 	at java.util.TreeMap.lastKey(TreeMap.java:297)
   [junit4]    > 	at java.util.TreeSet.last(TreeSet.java:401)
   [junit4]    > 	at org.apache.lucene.analysis.minhash.MinHashFilter$FixedSizeTreeSet.add(MinHashFilter.java:325)
   [junit4]    > 	at org.apache.lucene.analysis.minhash.MinHashFilter.incrementToken(MinHashFilter.java:159)
   [junit4]    > 	at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:67)
   [junit4]    > 	at org.apache.lucene.analysis.util.ElisionFilter.incrementToken(ElisionFilter.java:52)
   [junit4]    > 	at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:67)
   [junit4]    > 	at org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter.incrementToken(ASCIIFoldingFilter.java:102)
   [junit4]    > 	at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:67)
   [junit4]    > 	at org.apache.lucene.analysis.FilteringTokenFilter.incrementToken(FilteringTokenFilter.java:51)
   [junit4]    > 	at org.apache.lucene.analysis.ngram.NGramTokenFilter.incrementToken(NGramTokenFilter.java:98)
   [junit4]    > 	at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:67)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkResetException(BaseTokenStreamTestCase.java:405)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:510)
   [junit4]    > 	at org.apache.lucene.analysis.core.TestRandomChains.testRandomChainsWithLargeStrings(TestRandomChains.java:959)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/analysis/common/test/J0/temp/lucene.analysis.core.TestRandomChains_B9ED0ECBBD7CADDD-001
   [junit4]   2> NOTE: test params are: codec=DummyCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=DUMMY, chunkSize=8, maxDocsPerChunk=6, blockSize=176), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=DUMMY, chunkSize=8, blockSize=176)), sim=RandomSimilarity(queryNorm=false): {dummy=IB SPL-L3(800.0)}, locale=ru, timezone=Africa/Monrovia
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=93458712,total=296222720
   [junit4]   2> NOTE: All tests run in this JVM: [TestFrenchLightStemFilterFactory, TestDictionaryCompoundWordTokenFilterFactory, TestLimitTokenOffsetFilterFactory, TestHindiFilters, TestStemmer, TestPathHierarchyTokenizer, TestSpanishAnalyzer, TestFinnishLightStemFilterFactory, TestFullStrip, TestLengthFilter, TestCustomAnalyzer, TestGalicianStemFilterFactory, TestSwedishAnalyzer, TestDependencies, TestIgnore, TestTurkishAnalyzer, TestApostropheFilter, TestScandinavianFoldingFilter, TestHungarianLightStemFilterFactory, TestIndonesianStemmer, TestSpanishLightStemFilterFactory, TestElisionFilterFactory, WikipediaTokenizerTest, TestCollationKeyAnalyzer, TestGermanMinimalStemFilterFactory, TestTrimFilterFactory, TestIndonesianStemFilterFactory, TestNorwegianLightStemFilter, TestPorterStemFilter, TestRussianLightStemFilterFactory, TestCapitalizationFilter, TestArabicFilters, TestBasqueAnalyzer, TestCaseSensitive, TestCollationDocValuesField, QueryAutoStopWordAnalyzerTest, NGramTokenizerTest, TestGermanMinimalStemFilter, TestSoraniNormalizationFilterFactory, TestEnglishAnalyzer, TestPersianNormalizationFilter, TestNorwegianAnalyzer, TestCJKWidthFilter, TestDecimalDigitFilter, TestKeywordRepeatFilter, TestFactories, TestRandomChains]
   [junit4] Completed [122/272 (1!)] on J0 in 117.86s, 2 tests, 1 error <<< FAILURES!

[...truncated 8348 lines...]
   [junit4] Suite: org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/init-core-data-001
   [junit4]   2> 841040 INFO  (SUITE-ConcurrentDeleteAndCreateCollectionTest-seed#[69C7C3B4D964326A]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 841043 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.SolrTestCaseJ4 ###Starting testConcurrentCreateAndDeleteOverTheSameConfig
   [junit4]   2> 841044 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 841080 INFO  (Thread-953) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 841080 INFO  (Thread-953) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 841173 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.ZkTestServer start zk server on port:52340
   [junit4]   2> 841173 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 841225 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 841245 INFO  (zkCallback-503-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@1e554cc8 name:ZooKeeperConnection Watcher:127.0.0.1:52340 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 841245 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 841245 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 841245 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr/solr.xml
   [junit4]   2> 841247 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr/clusterprops.json
   [junit4]   2> 841255 INFO  (jetty-launcher-502-thread-1) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 841266 INFO  (jetty-launcher-502-thread-1) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@1a88dbc2{/solr,null,AVAILABLE}
   [junit4]   2> 841269 INFO  (jetty-launcher-502-thread-1) [    ] o.e.j.s.ServerConnector Started ServerConnector@1923033d{SSL,[ssl, http/1.1]}{127.0.0.1:44324}
   [junit4]   2> 841269 INFO  (jetty-launcher-502-thread-1) [    ] o.e.j.s.Server Started @849329ms
   [junit4]   2> 841269 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {hostContext=/solr, hostPort=44324}
   [junit4]   2> 841293 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 841293 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1'
   [junit4]   2> 841294 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 841294 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 841294 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 841309 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 841353 INFO  (zkCallback-504-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@c016771 name:ZooKeeperConnection Watcher:127.0.0.1:52340/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 841355 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 841355 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 841355 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.s.SolrDispatchFilter solr.xml found in ZooKeeper. Loading...
   [junit4]   2> 841369 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1/.
   [junit4]   2> 841369 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.CoreContainer New CoreContainer 866240293
   [junit4]   2> 841369 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1]
   [junit4]   2> 841369 WARN  (jetty-launcher-502-thread-1) [    ] o.a.s.c.CoreContainer Couldn't add files from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1/lib to classpath: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1/lib
   [junit4]   2> 841390 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : https,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,
   [junit4]   2> 841414 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.u.UpdateShardHandler Created UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&maxConnections=100000&maxConnectionsPerHost=100000
   [junit4]   2> 841414 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 841414 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 841414 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:52340/solr
   [junit4]   2> 841414 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 841414 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 841437 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 841452 INFO  (zkCallback-507-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6695662e name:ZooKeeperConnection Watcher:127.0.0.1:52340 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 841452 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 841452 INFO  (jetty-launcher-502-thread-1) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 841453 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$83/793159416@191f8e5d
   [junit4]   2> 841501 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 841517 INFO  (zkCallback-508-thread-1-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@513e6957 name:ZooKeeperConnection Watcher:127.0.0.1:52340/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 841517 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 841518 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
   [junit4]   2> 841520 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
   [junit4]   2> 841523 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
   [junit4]   2> 841524 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
   [junit4]   2> 841526 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
   [junit4]   2> 841527 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 841528 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections
   [junit4]   2> 841529 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 841530 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 841531 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 841532 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 841532 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ZkStateReader Loaded cluster properties: {urlScheme=https}
   [junit4]   2> 841533 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 841535 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
   [junit4]   2> 841536 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
   [junit4]   2> 841537 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 841538 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/96575435014995971-127.0.0.1:44324_solr-n_0000000000
   [junit4]   2> 841538 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:44324_solr
   [junit4]   2> 841538 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
   [junit4]   2> 841539 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.Overseer Overseer (id=96575435014995971-127.0.0.1:44324_solr-n_0000000000) starting
   [junit4]   2> 841541 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
   [junit4]   2> 841546 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 841561 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.Overseer Starting to work on the main queue
   [junit4]   2> 841562 INFO  (OverseerCollectionConfigSetProcessor-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
   [junit4]   2> 842034 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 842035 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44324_solr
   [junit4]   2> 842035 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:44324_solr
   [junit4]   2> 842037 INFO  (zkCallback-508-thread-1-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 842037 INFO  (zkCallback-508-thread-1-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 842200 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 842200 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 842225 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1/.
   [junit4]   2> 842226 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.c.CorePropertiesLocator Found 0 core definitions
   [junit4]   2> 842233 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.s.SolrDispatchFilter user.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1
   [junit4]   2> 842233 INFO  (jetty-launcher-502-thread-1) [n:127.0.0.1:44324_solr    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 842233 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 842245 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 842257 INFO  (zkCallback-510-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@64b90cc9 name:ZooKeeperConnection Watcher:127.0.0.1:52340 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 842257 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 842257 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 842262 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 842300 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 842310 INFO  (zkCallback-512-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@1c09f4f2 name:ZooKeeperConnection Watcher:127.0.0.1:52340/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 842310 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 842311 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 842311 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/testconfig/schema.xml
   [junit4]   2> 842314 INFO  (TEST-ConcurrentDeleteAndCreateCollectionTest.testConcurrentCreateAndDeleteOverTheSameConfig-seed#[69C7C3B4D964326A]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/testconfig/solrconfig.xml
   [junit4]   2> 842505 INFO  (qtp946450217-3432) [n:127.0.0.1:44324_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=testconfig&name=collection0&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 842508 INFO  (OverseerCollectionConfigSetProcessor-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000000 message:{
   [junit4]   2>   "name":"collection0",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"testconfig",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 842509 INFO  (OverseerThreadFactory-1293-thread-1-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"collection0",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"testconfig",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 842510 INFO  (OverseerThreadFactory-1293-thread-1-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection0
   [junit4]   2> 842510 INFO  (qtp946450217-3433) [n:127.0.0.1:44324_solr    ] o.a.s.h.a.CollectionsHandler Invoked Collection Action :create with params replicationFactor=1&collection.configName=testconfig&name=collection1&action=CREATE&numShards=1&wt=javabin&version=2 and sendToOCPQueue=true
   [junit4]   2> 842517 INFO  (OverseerThreadFactory-1293-thread-1-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection0 
   [junit4]   2> 842533 INFO  (OverseerThreadFactory-1293-thread-1-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection0
   [junit4]   2> 842538 INFO  (OverseerCollectionConfigSetProcessor-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.OverseerTaskProcessor Overseer Collection Message Handler: Get the message id:/overseer/collection-queue-work/qn-0000000002 message:{
   [junit4]   2>   "name":"collection1",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"testconfig",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 842540 INFO  (OverseerThreadFactory-1293-thread-2-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.OverseerCollectionMessageHandler OverseerCollectionMessageHandler.processMessage : create , {
   [junit4]   2>   "name":"collection1",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"testconfig",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"}
   [junit4]   2> 842540 INFO  (OverseerThreadFactory-1293-thread-2-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.CreateCollectionCmd Create collection collection1
   [junit4]   2> 842541 INFO  (OverseerThreadFactory-1293-thread-2-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.OverseerCollectionMessageHandler creating collections conf node /collections/collection1 
   [junit4]   2> 842545 INFO  (zkCallback-508-thread-1-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 842545 INFO  (OverseerThreadFactory-1293-thread-2-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1
   [junit4]   2> 842562 INFO  (zkCallback-508-thread-2-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ZkStateReader A collections change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/collections], has occurred - updating...
   [junit4]   2> 842580 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
   [junit4]   2>   "name":"collection1",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"testconfig",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 842580 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection1
   [junit4]   2> 842586 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection1/state.json
   [junit4]   2> 842588 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "name":"collection0",
   [junit4]   2>   "fromApi":"true",
   [junit4]   2>   "replicationFactor":"1",
   [junit4]   2>   "collection.configName":"testconfig",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "stateFormat":"2",
   [junit4]   2>   "operation":"create"} current state version: 0
   [junit4]   2> 842589 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.o.ClusterStateMutator building a new cName: collection0
   [junit4]   2> 842777 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.o.ZkStateWriter going to create_collection /collections/collection0/state.json
   [junit4]   2> 842778 INFO  (OverseerThreadFactory-1293-thread-2-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.CreateCollectionCmd Creating SolrCores for new collection collection1, shardNames [shard1] , replicationFactor : 1
   [junit4]   2> 842778 INFO  (OverseerThreadFactory-1293-thread-2-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.CreateCollectionCmd Creating core collection1_shard1_replica1 as part of shard shard1 of collection collection1 on 127.0.0.1:44324_solr
   [junit4]   2> 842885 INFO  (OverseerThreadFactory-1293-thread-1-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.CreateCollectionCmd Creating SolrCores for new collection collection0, shardNames [shard1] , replicationFactor : 1
   [junit4]   2> 842885 INFO  (OverseerThreadFactory-1293-thread-1-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.CreateCollectionCmd Creating core collection0_shard1_replica1 as part of shard shard1 of collection collection0 on 127.0.0.1:44324_solr
   [junit4]   2> 842962 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=testconfig&name=collection0_shard1_replica1&action=CREATE&numShards=1&collection=collection0&shard=shard1&wt=javabin&version=2
   [junit4]   2> 842964 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection0_shard1_replica1, collection.configName=testconfig, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.json, collection=collection0, numShards=1, dataDir=data/, shard=shard1}
   [junit4]   2> 842964 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.ZkController publishing state=down
   [junit4]   2> 842965 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.ZkController Registering watch for collection collection0
   [junit4]   2> 842966 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.c.ZkStateReader Add data for [collection0] ver [0]
   [junit4]   2> 842966 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.ZkController look for our core node name
   [junit4]   2> 842978 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr    ] o.a.s.h.a.CoreAdminOperation core create command qt=/admin/cores&collection.configName=testconfig&name=collection1_shard1_replica1&action=CREATE&numShards=1&collection=collection1&shard=shard1&wt=javabin&version=2
   [junit4]   2> 842978 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection1_shard1_replica1, collection.configName=testconfig, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.json, collection=collection1, numShards=1, dataDir=data/, shard=shard1}
   [junit4]   2> 842979 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.ZkController publishing state=down
   [junit4]   2> 842980 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.ZkController Registering watch for collection collection1
   [junit4]   2> 842980 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.c.ZkStateReader Add data for [collection1] ver [0]
   [junit4]   2> 842980 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.ZkController look for our core node name
   [junit4]   2> 842997 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
   [junit4]   2>   "core":"collection0_shard1_replica1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"https://127.0.0.1:44324/solr",
   [junit4]   2>   "node_name":"127.0.0.1:44324_solr",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"collection0",
   [junit4]   2>   "operation":"state"} current state version: 0
   [junit4]   2> 842997 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
   [junit4]   2>   "core":"collection0_shard1_replica1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"https://127.0.0.1:44324/solr",
   [junit4]   2>   "node_name":"127.0.0.1:44324_solr",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"collection0",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 843002 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.Overseer processMessage: queueSize: 2, message = {
   [junit4]   2>   "core":"collection1_shard1_replica1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"https://127.0.0.1:44324/solr",
   [junit4]   2>   "node_name":"127.0.0.1:44324_solr",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "operation":"state"} current state version: 0
   [junit4]   2> 843003 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
   [junit4]   2>   "core":"collection1_shard1_replica1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"https://127.0.0.1:44324/solr",
   [junit4]   2>   "node_name":"127.0.0.1:44324_solr",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":"shard1",
   [junit4]   2>   "collection":"collection1",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 843003 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/collection0/state.json version: 0
   [junit4]   2> 843005 INFO  (zkCallback-508-thread-4-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection0/state.json] for collection [collection0] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 843009 INFO  (zkCallback-508-thread-4-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ZkStateReader Updating data for [collection0] from [0] to [1]
   [junit4]   2> 843178 INFO  (OverseerStateUpdate-96575435014995971-127.0.0.1:44324_solr-n_0000000000) [n:127.0.0.1:44324_solr    ] o.a.s.c.o.ZkStateWriter going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 843179 INFO  (zkCallback-508-thread-3-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json] for collection [collection1] has occurred - updating... (live nodes size: [1])
   [junit4]   2> 843179 INFO  (zkCallback-508-thread-3-processing-n:127.0.0.1:44324_solr) [n:127.0.0.1:44324_solr    ] o.a.s.c.c.ZkStateReader Updating data for [collection1] from [0] to [1]
   [junit4]   2> 844030 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.ZkController waiting to find shard id in clusterstate for collection0_shard1_replica1
   [junit4]   2> 844030 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.ZkController Check for collection zkNode:collection0
   [junit4]   2> 844030 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.ZkController waiting to find shard id in clusterstate for collection1_shard1_replica1
   [junit4]   2> 844030 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.ZkController Check for collection zkNode:collection1
   [junit4]   2> 844031 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.ZkController Collection zkNode exists
   [junit4]   2> 844032 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.c.ZkStateReader Load collection config from: [/collections/collection1]
   [junit4]   2> 844032 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.ZkController Collection zkNode exists
   [junit4]   2> 844032 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.c.ZkStateReader Load collection config from: [/collections/collection0]
   [junit4]   2> 844033 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.c.ZkStateReader path=[/collections/collection1] [configName]=[testconfig] specified config exists in ZooKeeper
   [junit4]   2> 844033 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1/collection1_shard1_replica1'
   [junit4]   2> 844034 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 844034 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 844034 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.c.ZkStateReader path=[/collections/collection0] [configName]=[testconfig] specified config exists in ZooKeeper
   [junit4]   2> 844035 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1/collection0_shard1_replica1'
   [junit4]   2> 844035 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 844035 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 844036 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.Config loaded config solrconfig.xml with version 0 
   [junit4]   2> 844037 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.Config loaded config solrconfig.xml with version 0 
   [junit4]   2> 844042 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 844043 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 844050 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 844061 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 844094 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 844107 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 844119 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.s.IndexSchema [collection0_shard1_replica1] Schema name=minimal
   [junit4]   2> 844122 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 844124 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.IndexSchema [collection1_shard1_replica1] Schema name=minimal
   [junit4]   2> 844126 INFO  (qtp946450217-3438) [n:127.0.0.1:44324_solr c:collection1 s:shard1  x:collection1_shard1_replica1] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 844126 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properties: Can't find resource 'configsetprops.json' in classpath or '/configs/testconfig', cwd=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1
   [junit4]   2> 844126 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1  x:collection0_shard1_replica1] o.a.s.c.CoreContainer Creating SolrCore 'collection0_shard1_replica1' using configuration from collection collection0
   [junit4]   2> 844127 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 844129 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore [[collection0_shard1_replica1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1/collection0_shard1_replica1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1/./collection0_shard1_replica1/data/]
   [junit4]   2> 844129 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore JMX monitoring not detected for core: collection0_shard1_replica1
   [junit4]   2> 844129 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1/./collection0_shard1_replica1/data/snapshot_metadata
   [junit4]   2> 844129 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.s.SolrSnapshotMetaDataManager Loading from snapshot metadata file...
   [junit4]   2> 844130 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1/./collection0_shard1_replica1/data
   [junit4]   2> 844130 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore New index directory detected: old=null new=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1/./collection0_shard1_replica1/data/index/
   [junit4]   2> 844130 WARN  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.SolrCore [collection0_shard1_replica1] Solr index directory '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.ConcurrentDeleteAndCreateCollectionTest_69C7C3B4D964326A-001/tempDir-001/node1/./collection0_shard1_replica1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 844130 INFO  (qtp946450217-3437) [n:127.0.0.1:44324_solr c:collection0 s:shard1 r:core_node1 x:collection0_shard1_replica1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-sla

[...truncated too long message...]

p/solr.handler.TestSolrConfigHandlerCloud_69C7C3B4D964326A-001/shard-4-001/cores/collection1/data/snapshot_metadata
   [junit4]   2> 7363461 INFO  (Thread-87801) [n:127.0.0.1:49246_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.CachingDirectoryFactory looking to close /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestSolrConfigHandlerCloud_69C7C3B4D964326A-001/shard-4-001/cores/collection1/data [CachedDir<<refCount=0;path=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestSolrConfigHandlerCloud_69C7C3B4D964326A-001/shard-4-001/cores/collection1/data;done=false>>]
   [junit4]   2> 7363461 INFO  (Thread-87801) [n:127.0.0.1:49246_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.CachingDirectoryFactory Closing directory, CoreContainer#isShutdown=true
   [junit4]   2> 7363461 INFO  (Thread-87801) [n:127.0.0.1:49246_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.CachingDirectoryFactory Closing directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestSolrConfigHandlerCloud_69C7C3B4D964326A-001/shard-4-001/cores/collection1/data
   [junit4]   2> 7363467 INFO  (Thread-87801) [n:127.0.0.1:49246_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.CachingDirectoryFactory looking to close /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestSolrConfigHandlerCloud_69C7C3B4D964326A-001/shard-4-001/cores/collection1/data/index [CachedDir<<refCount=0;path=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestSolrConfigHandlerCloud_69C7C3B4D964326A-001/shard-4-001/cores/collection1/data/index;done=false>>]
   [junit4]   2> 7363467 INFO  (Thread-87801) [n:127.0.0.1:49246_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.CachingDirectoryFactory Closing directory, CoreContainer#isShutdown=true
   [junit4]   2> 7363467 INFO  (Thread-87801) [n:127.0.0.1:49246_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.CachingDirectoryFactory Closing directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestSolrConfigHandlerCloud_69C7C3B4D964326A-001/shard-4-001/cores/collection1/data/index
   [junit4]   2> 7363467 WARN  (Thread-87801) [n:127.0.0.1:49246_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.SolrCores Tried to remove core collection1 from pendingCoreOps and it wasn't there. 
   [junit4]   2> 7363467 WARN  (Thread-87801) [n:127.0.0.1:49246_ c:collection1 s:shard1 r:core_node4 x:collection1] o.a.s.c.ZkController listener throws error
   [junit4]   2> org.apache.solr.common.SolrException: Unable to reload core [collection1]
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:968)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.lambda$getConfListener$6(SolrCore.java:2602)
   [junit4]   2> 	at org.apache.solr.cloud.ZkController.lambda$fireEventListeners$3(ZkController.java:2443)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Failed to load config name for collection:collection1 due to: 
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:833)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.reload(SolrCore.java:503)
   [junit4]   2> 	at org.apache.solr.core.CoreContainer.reload(CoreContainer.java:962)
   [junit4]   2> 	... 3 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Failed to load config name for collection:collection1 due to: 
   [junit4]   2> 	at org.apache.solr.rest.ManagedResourceStorage.newStorageIO(ManagedResourceStorage.java:98)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.initRestManager(SolrCore.java:2382)
   [junit4]   2> 	at org.apache.solr.core.SolrCore.<init>(SolrCore.java:809)
   [junit4]   2> 	... 5 more
   [junit4]   2> Caused by: org.apache.solr.common.SolrException: Error loading config name for collection collection1
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.readConfigName(ZkStateReader.java:197)
   [junit4]   2> 	at org.apache.solr.rest.ManagedResourceStorage.newStorageIO(ManagedResourceStorage.java:95)
   [junit4]   2> 	... 7 more
   [junit4]   2> Caused by: org.apache.zookeeper.KeeperException$SessionExpiredException: KeeperErrorCode = Session expired for /collections/collection1
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:127)
   [junit4]   2> 	at org.apache.zookeeper.KeeperException.create(KeeperException.java:51)
   [junit4]   2> 	at org.apache.zookeeper.ZooKeeper.getData(ZooKeeper.java:1155)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:348)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient$7.execute(SolrZkClient.java:345)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkCmdExecutor.retryOperation(ZkCmdExecutor.java:60)
   [junit4]   2> 	at org.apache.solr.common.cloud.SolrZkClient.getData(SolrZkClient.java:345)
   [junit4]   2> 	at org.apache.solr.common.cloud.ZkStateReader.readConfigName(ZkStateReader.java:178)
   [junit4]   2> 	... 8 more
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {}, docValues:{}, maxPointsInLeafNode=238, maxMBSortInHeap=5.750380438790871, sim=RandomSimilarity(queryNorm=false): {}, locale=ko, timezone=Africa/Niamey
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=157937416,total=521142272
   [junit4]   2> NOTE: All tests run in this JVM: [TestSolrConfigHandler, TestLegacyNumericRangeQueryBuilder, AnalyticsQueryTest, ResourceLoaderTest, TestRandomCollapseQParserPlugin, TestDFISimilarityFactory, PeerSyncTest, SmileWriterTest, TestSortByMinMaxFunction, RankQueryTest, DateFieldTest, LeaderElectionIntegrationTest, TestFieldTypeCollectionResource, DistributedExpandComponentTest, QueryEqualityTest, TestJsonFacets, QueryElevationComponentTest, RestartWhileUpdatingTest, SpatialRPTFieldTypeTest, TestSolrDeletionPolicy2, SolrRequestParserTest, HdfsTlogReplayBufferedWhileIndexingTest, UUIDUpdateProcessorFallbackTest, OverseerTaskQueueTest, CacheHeaderTest, TestDocTermOrds, CachingDirectoryFactoryTest, TestSimpleQParserPlugin, TestSerializedLuceneMatchVersion, SolrCloudExampleTest, TestConfigSetProperties, TestReplicationHandler, CloudMLTQParserTest, SignatureUpdateProcessorFactoryTest, XsltUpdateRequestHandlerTest, TestOmitPositions, ConnectionManagerTest, HdfsNNFailoverTest, DistributedFacetPivotLongTailTest, TestStressRecovery, TestRangeQuery, SecurityConfHandlerTest, RequiredFieldsTest, DataDrivenBlockJoinTest, TestHdfsUpdateLog, DistribCursorPagingTest, TestImplicitCoreProperties, TestComponentsName, HttpPartitionTest, TestUniqueKeyFieldResource, BlockJoinFacetRandomTest, TestCrossCoreJoin, TestConfigSetsAPIExclusivity, DistributedSuggestComponentTest, OverseerModifyCollectionTest, TestPHPSerializedResponseWriter, TestAtomicUpdateErrorCases, TestHighlightDedupGrouping, StandardRequestHandlerTest, DateRangeFieldTest, CollectionsAPISolrJTest, TestReplicaProperties, TestSolr4Spatial, SyncSliceTest, AsyncCallRequestStatusResponseTest, ZkStateWriterTest, TestExactSharedStatsCache, TestScoreJoinQPNoScore, TestJoin, TestManagedSchema, TestDocumentBuilder, RegexBoostProcessorTest, TestHighFrequencyDictionaryFactory, SuggestComponentContextFilterQueryTest, TestUseDocValuesAsStored, JavabinLoaderTest, DeleteReplicaTest, TestJmxIntegration, TestExpandComponent, TestHdfsCloudBackupRestore, StressHdfsTest, TestSSLRandomization, AlternateDirectoryTest, DistributedQueueTest, TestDynamicFieldResource, TestSolrCoreProperties, HLLUtilTest, TestConfigReload, TestDistributedSearch, TestGraphTermsQParserPlugin, TestReqParamsAPI, ResponseHeaderTest, TestValueSourceCache, TestQuerySenderNoQuery, SaslZkACLProviderTest, TestFoldingMultitermQuery, PathHierarchyTokenizerFactoryTest, CursorPagingTest, TestSystemIdResolver, SpellingQueryConverterTest, ClusterStateTest, ZkNodePropsTest, SliceStateTest, SystemInfoHandlerTest, UUIDFieldTest, DistributedMLTComponentTest, TestRTGBase, DistributedIntervalFacetingTest, TestEmbeddedSolrServerConstructors, TestJettySolrRunner, ActionThrottleTest, AsyncMigrateRouteKeyTest, CdcrReplicationDistributedZkTest, TestStressLiveNodes, TestBackupRepositoryFactory, TestConfigOverlay, TestConfigSets, TestNRTOpen, RequestLoggingTest, TestBlobHandler, TestSolrConfigHandlerCloud]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSolrConfigHandlerCloud -Dtests.seed=69C7C3B4D964326A -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=ko -Dtests.timezone=Africa/Niamey -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J0 | TestSolrConfigHandlerCloud (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: ObjectTracker found 8 object(s) that were not released!!! [MDCAwareThreadPoolExecutor, MockDirectoryWrapper, SolrCore, MockDirectoryWrapper, MDCAwareThreadPoolExecutor, SolrCore, MDCAwareThreadPoolExecutor, MockDirectoryWrapper]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([69C7C3B4D964326A]:0)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4] Completed [544/631 (6!)] on J0 in 114.89s, 1 test, 1 failure <<< FAILURES!

[...truncated 60452 lines...]



[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1113 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1113/

13 tests failed.
FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard2

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard2
	at __randomizedtesting.SeedInfo.seed([8F24B045D9638E9:54EFAC8593B68CDF]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart(CdcrReplicationDistributedZkTest.java:236)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.DistributedVersionInfoTest.testReplicaVersionHandling

Error Message:
Captured an uncaught exception in thread: Thread[id=169355, name=Thread-5485, state=RUNNABLE, group=TGRP-DistributedVersionInfoTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=169355, name=Thread-5485, state=RUNNABLE, group=TGRP-DistributedVersionInfoTest]
	at __randomizedtesting.SeedInfo.seed([8F24B045D9638E9:D40B9CFEFFEDF2A8]:0)
Caused by: java.lang.IllegalArgumentException: bound must be positive
	at __randomizedtesting.SeedInfo.seed([8F24B045D9638E9]:0)
	at java.util.Random.nextInt(Random.java:388)
	at org.apache.solr.cloud.DistributedVersionInfoTest$3.run(DistributedVersionInfoTest.java:204)


FAILED:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:36753/na/collection2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:36753/na/collection2
	at __randomizedtesting.SeedInfo.seed([8F24B045D9638E9:80A674DEF36A5511]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:415)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:367)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1280)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1050)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:992)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:484)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:463)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.commit(AbstractFullDistribZkTestBase.java:1530)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:664)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test(FullSolrCloudDistribCmdsTest.java:152)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 54 more


FAILED:  org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test

Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:38312/solrj_collection4, http://127.0.0.1:58988/solrj_collection4, http://127.0.0.1:43269/solrj_collection4]

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:38312/solrj_collection4, http://127.0.0.1:58988/solrj_collection4, http://127.0.0.1:43269/solrj_collection4]
	at __randomizedtesting.SeedInfo.seed([8F24B045D9638E9:80A674DEF36A5511]:0)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:394)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1280)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1050)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:992)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.queryAndAssertResultSize(SharedFSAutoReplicaFailoverTest.java:326)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.testBasics(SharedFSAutoReplicaFailoverTest.java:260)
	at org.apache.solr.cloud.SharedFSAutoReplicaFailoverTest.test(SharedFSAutoReplicaFailoverTest.java:127)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:43269/solrj_collection4: no servers hosting shard: shard2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:608)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:415)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:367)
	... 50 more


FAILED:  junit.framework.TestSuite.org.apache.solr.handler.TestSolrConfigHandlerCloud

Error Message:
ObjectTracker found 4 object(s) that were not released!!! [MockDirectoryWrapper, MockDirectoryWrapper, MockDirectoryWrapper, MDCAwareThreadPoolExecutor]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 4 object(s) that were not released!!! [MockDirectoryWrapper, MockDirectoryWrapper, MockDirectoryWrapper, MDCAwareThreadPoolExecutor]
	at __randomizedtesting.SeedInfo.seed([8F24B045D9638E9]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.GeneratedMethodAccessor66.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest.test

Error Message:
Error from server at http://127.0.0.1:33658/p/c: Error CREATEing SolrCore 'test_unload_shard_and_collection_2': Unable to create core [test_unload_shard_and_collection_2] Caused by: Direct buffer memory

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:33658/p/c: Error CREATEing SolrCore 'test_unload_shard_and_collection_2': Unable to create core [test_unload_shard_and_collection_2] Caused by: Direct buffer memory
	at __randomizedtesting.SeedInfo.seed([8F24B045D9638E9:80A674DEF36A5511]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:608)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.UnloadDistributedZkTest.testUnloadShardAndCollection(UnloadDistributedZkTest.java:137)
	at org.apache.solr.cloud.UnloadDistributedZkTest.test(UnloadDistributedZkTest.java:70)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigAliasReplication

Error Message:
expected:<1> but was:<0>

Stack Trace:
java.lang.AssertionError: expected:<1> but was:<0>
	at __randomizedtesting.SeedInfo.seed([8F24B045D9638E9:FF81A55C9B7E970F]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigAliasReplication(TestReplicationHandler.java:1329)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestStressReplication

Error Message:
timed out waiting for collection1 startAt time to exceed: Sat Sep 10 12:13:15 EDT 2016

Stack Trace:
java.lang.AssertionError: timed out waiting for collection1 startAt time to exceed: Sat Sep 10 12:13:15 EDT 2016
	at __randomizedtesting.SeedInfo.seed([8F24B045D9638E9:D3594BC258BE515A]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.handler.TestReplicationHandler.watchCoreStartAt(TestReplicationHandler.java:1508)
	at org.apache.solr.handler.TestReplicationHandler.doTestStressReplication(TestReplicationHandler.java:858)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchAddsWithDelete

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard2

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard2
	at __randomizedtesting.SeedInfo.seed([8F24B045D9638E9:7FE832A0F58F39C5]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchAddsWithDelete(CdcrReplicationDistributedZkTest.java:526)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:50004

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:50004
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:497)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:44483

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:44483
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:516)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
	at __randomizedtesting.SeedInfo.seed([8F24B045D9638E9]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.GeneratedMethodAccessor66.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest:     1) Thread[id=70872, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=70862, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=70907, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=70887, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=70897, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=70867, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=70882, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=70877, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=70902, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=70892, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 
   1) Thread[id=70872, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=70862, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=70907, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=70887, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=70897, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=70867, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=70882, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=70877, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=70902, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=70892, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([8F24B045D9638E9]:0)




Build Log:
[...truncated 11384 lines...]
   [junit4] Suite: org.apache.solr.cloud.FullSolrCloudDistribCmdsTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_8F24B045D9638E9-001/init-core-data-001
   [junit4]   2> 547883 INFO  (SUITE-FullSolrCloudDistribCmdsTest-seed#[8F24B045D9638E9]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 547884 INFO  (SUITE-FullSolrCloudDistribCmdsTest-seed#[8F24B045D9638E9]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /na/
   [junit4]   2> 547885 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 547935 INFO  (Thread-1450) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 547935 INFO  (Thread-1450) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 548039 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.ZkTestServer start zk server on port:48470
   [junit4]   2> 548039 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 548063 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 548143 INFO  (zkCallback-502-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@34a87f8d name:ZooKeeperConnection Watcher:127.0.0.1:48470 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 548147 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 548147 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 548147 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 548150 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 548151 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 548153 INFO  (zkCallback-503-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@7dc1d05e name:ZooKeeperConnection Watcher:127.0.0.1:48470/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 548153 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 548153 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 548153 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1
   [junit4]   2> 548154 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/shards
   [junit4]   2> 548218 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection
   [junit4]   2> 548219 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection/shards
   [junit4]   2> 548220 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 548220 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 548222 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 548222 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/schema.xml
   [junit4]   2> 548223 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 548223 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 548224 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 548225 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 548226 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 548226 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/protwords.txt
   [junit4]   2> 548227 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 548227 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/currency.xml
   [junit4]   2> 548228 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 548228 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 548229 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 548229 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 548230 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 548231 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 548232 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 548232 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 548233 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 548233 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 548440 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_8F24B045D9638E9-001/control-001/cores/collection1
   [junit4]   2> 548442 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 548477 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@59d040ea{/na,null,AVAILABLE}
   [junit4]   2> 548494 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.e.j.s.ServerConnector Started ServerConnector@30567dde{HTTP/1.1,[http/1.1]}{127.0.0.1:33093}
   [junit4]   2> 548495 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.e.j.s.Server Started @556035ms
   [junit4]   2> 548495 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_8F24B045D9638E9-001/tempDir-001/control/data, hostContext=/na, hostPort=33093, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_8F24B045D9638E9-001/control-001/cores}
   [junit4]   2> 548499 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 548499 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_8F24B045D9638E9-001/control-001'
   [junit4]   2> 548499 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 548499 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 548500 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 548531 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 548531 INFO  (zkCallback-504-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@3704b9ac name:ZooKeeperConnection Watcher:127.0.0.1:48470/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 548531 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 548532 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 548543 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 548543 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_8F24B045D9638E9-001/control-001/solr.xml
   [junit4]   2> 548548 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_8F24B045D9638E9-001/control-001/cores
   [junit4]   2> 548548 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.CoreContainer New CoreContainer 703387634
   [junit4]   2> 548548 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_8F24B045D9638E9-001/control-001]
   [junit4]   2> 548548 WARN  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.CoreContainer Couldn't add files from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_8F24B045D9638E9-001/control-001/lib to classpath: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_8F24B045D9638E9-001/control-001/lib
   [junit4]   2> 548549 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,
   [junit4]   2> 548603 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.u.UpdateShardHandler Created UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&maxConnections=100000&maxConnectionsPerHost=100000
   [junit4]   2> 548603 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 548603 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 548603 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:48470/solr
   [junit4]   2> 548604 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 548604 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 548619 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 548631 INFO  (zkCallback-507-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4f62f01f name:ZooKeeperConnection Watcher:127.0.0.1:48470 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 548636 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 548636 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 548637 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$99/841033882@11d68def
   [junit4]   2> 548804 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 548816 INFO  (zkCallback-508-thread-1-processing-n:127.0.0.1:33093_na) [n:127.0.0.1:33093_na    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6d43343f name:ZooKeeperConnection Watcher:127.0.0.1:48470/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 548816 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 548817 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
   [junit4]   2> 548819 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
   [junit4]   2> 548821 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
   [junit4]   2> 548822 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
   [junit4]   2> 548823 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
   [junit4]   2> 548825 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 548826 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 548826 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 548827 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 548828 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 548828 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 548829 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 548831 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
   [junit4]   2> 548832 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
   [junit4]   2> 548833 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 548833 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/96568706267938820-127.0.0.1:33093_na-n_0000000000
   [junit4]   2> 548833 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:33093_na
   [junit4]   2> 548833 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
   [junit4]   2> 548834 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.Overseer Overseer (id=96568706267938820-127.0.0.1:33093_na-n_0000000000) starting
   [junit4]   2> 548835 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
   [junit4]   2> 548839 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=10000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 548847 INFO  (OverseerStateUpdate-96568706267938820-127.0.0.1:33093_na-n_0000000000) [n:127.0.0.1:33093_na    ] o.a.s.c.Overseer Starting to work on the main queue
   [junit4]   2> 548848 INFO  (OverseerCollectionConfigSetProcessor-96568706267938820-127.0.0.1:33093_na-n_0000000000) [n:127.0.0.1:33093_na    ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
   [junit4]   2> 548864 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:33093_na
   [junit4]   2> 548864 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:33093_na
   [junit4]   2> 548865 INFO  (OverseerStateUpdate-96568706267938820-127.0.0.1:33093_na-n_0000000000) [n:127.0.0.1:33093_na    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 548866 INFO  (zkCallback-508-thread-1-processing-n:127.0.0.1:33093_na) [n:127.0.0.1:33093_na    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 548867 INFO  (zkCallback-508-thread-1-processing-n:127.0.0.1:33093_na) [n:127.0.0.1:33093_na    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 548904 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 548904 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 548905 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J2/temp/solr.cloud.FullSolrCloudDistribCmdsTest_8F24B045D9638E9-001/control-001/cores
   [junit4]   2> 548906 INFO  (TEST-FullSolrCloudDistribCmdsTest.test-seed#[8F24B045D9638E9]) [n:127.0.0.1:33093_na    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.json, collection=control_collection, coreNodeName=, dataDir=data/, shard=}
  

[...truncated too long message...]

un.jmx.interceptor.DefaultMBeanServerInterceptor.exclusiveUnregisterMBean(DefaultMBeanServerInterceptor.java:430)
   [junit4]   2> 	at com.sun.jmx.interceptor.DefaultMBeanServerInterceptor.unregisterMBean(DefaultMBeanServerInterceptor.java:415)
   [junit4]   2> 	at com.sun.jmx.mbeanserver.JmxMBeanServer.unregisterMBean(JmxMBeanServer.java:546)
   [junit4]   2> 	at org.apache.hadoop.metrics2.util.MBeans.unregister(MBeans.java:81)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stopMBeans(MetricsSourceAdapter.java:226)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSourceAdapter.stop(MetricsSourceAdapter.java:211)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stopSources(MetricsSystemImpl.java:463)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.stop(MetricsSystemImpl.java:213)
   [junit4]   2> 	at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.shutdown(MetricsSystemImpl.java:594)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdownInstance(DefaultMetricsSystem.java:72)
   [junit4]   2> 	at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.shutdown(DefaultMetricsSystem.java:68)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics.shutdown(NameNodeMetrics.java:171)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.NameNode.stop(NameNode.java:872)
   [junit4]   2> 	at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1726)
   [junit4]   2> 	at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:1705)
   [junit4]   2> 	at org.apache.solr.cloud.hdfs.HdfsTestUtil.teardownClass(HdfsTestUtil.java:198)
   [junit4]   2> 	at org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.teardownClass(HdfsCollectionsAPIDistributedZkTest.java:52)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 	at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> Caused by: java.lang.NullPointerException
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlocksMap.size(BlocksMap.java:203)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.blockmanagement.BlockManager.getTotalBlocks(BlockManager.java:3370)
   [junit4]   2> 	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlocksTotal(FSNamesystem.java:5729)
   [junit4]   2> 	... 54 more
   [junit4]   2> 13249318 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[8F24B045D9638E9]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_8F24B045D9638E9-001
   [junit4]   2> Sep 10, 2016 7:03:41 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 35 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=FastDecompressionCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=8, maxDocsPerChunk=237, blockSize=396), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST_DECOMPRESSION, chunkSize=8, blockSize=396)), sim=RandomSimilarity(queryNorm=false): {}, locale=es-CL, timezone=Antarctica/Casey
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=2,free=141441624,total=516423680
   [junit4]   2> NOTE: All tests run in this JVM: [TestBulkSchemaConcurrent, TestFieldCollectionResource, TestFieldCache, CdcrReplicationDistributedZkTest, DirectSolrConnectionTest, SolrIndexConfigTest, ClassificationUpdateProcessorFactoryTest, TestLRUCache, DeleteShardTest, HdfsBasicDistributedZk2Test, TestDefaultSearchFieldResource, TestRandomRequestDistribution, SimpleCollectionCreateDeleteTest, DocumentBuilderTest, TestNamedUpdateProcessors, TestHashPartitioner, TestSolrConfigHandlerConcurrent, TestBM25SimilarityFactory, TestDistributedGrouping, TestFastOutputStream, SolrIndexSplitterTest, TestSchemaSimilarityResource, TestCSVResponseWriter, TestFieldCacheWithThreads, TestMiniSolrCloudClusterSSL, ShardRoutingCustomTest, FileUtilsTest, TestSystemIdResolver, TermsComponentTest, TestMinMaxOnMultiValuedField, TestSolrCloudWithKerberosAlt, TlogReplayBufferedWhileIndexingTest, AliasIntegrationTest, BlockCacheTest, TestSolrQueryResponse, DirectSolrSpellCheckerTest, TestScoreJoinQPScore, MultiTermTest, TestFieldSortValues, TestGraphMLResponseWriter, MigrateRouteKeyTest, SimplePostToolTest, CreateCollectionCleanupTest, CSVRequestHandlerTest, TestSurroundQueryParser, TestQueryTypes, TestUpdate, SampleTest, CursorMarkTest, TestRuleBasedAuthorizationPlugin, URLClassifyProcessorTest, TestManagedStopFilterFactory, TestRequestForwarding, TestFiltering, BooleanFieldTest, SpellCheckCollatorTest, LeaderFailoverAfterPartitionTest, BigEndianAscendingWordSerializerTest, TestFieldTypeResource, SolrXmlInZkTest, FullHLLTest, DistanceUnitsTest, CollectionsAPIDistributedZkTest, DeleteLastCustomShardedReplicaTest, BlockJoinFacetSimpleTest, SliceStateTest, TestSweetSpotSimilarityFactory, SpellingQueryConverterTest, CollectionTooManyReplicasTest, TestOverriddenPrefixQueryForCustomFieldType, CoreSorterTest, PKIAuthenticationIntegrationTest, TestReloadAndDeleteDocs, DistributedFacetPivotSmallAdvancedTest, FacetPivotSmallTest, TestInfoStreamLogging, DistributedVersionInfoTest, DistribDocExpirationUpdateProcessorTest, ReplaceNodeTest, TestLMDirichletSimilarityFactory, SpatialFilterTest, RequestLoggingTest, SolrCmdDistributorTest, SparseHLLTest, SearchHandlerTest, AsyncMigrateRouteKeyTest, TestManagedSchemaAPI, MoreLikeThisHandlerTest, OverriddenZkACLAndCredentialsProvidersTest, HdfsSyncSliceTest, CoreMergeIndexesAdminHandlerTest, DistanceFunctionTest, IndexSchemaRuntimeFieldTest, AnalysisErrorHandlingTest, SortByFunctionTest, RAMDirectoryFactoryTest, TestCloudInspectUtil, HdfsChaosMonkeySafeLeaderTest, SoftAutoCommitTest, SegmentsInfoRequestHandlerTest, SchemaVersionSpecificBehaviorTest, TestRTGBase, ConvertedLegacyTest, TestSolrCLIRunExample, HdfsCollectionsAPIDistributedZkTest]
   [junit4] Completed [631/631 (10!)] on J1 in 7031.62s, 1 test, 1 error <<< FAILURES!

[...truncated 1 lines...]
   [junit4] JVM J1: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20160910_152243_423.sysout
   [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: Java heap space
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid6780.hprof ...
   [junit4] Heap dump file created [722169601 bytes in 63.845 secs]
   [junit4] <<< JVM J1: EOF ----

[...truncated 11040 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:715: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid6762.hprof
* java_pid6780.hprof

Total time: 506 minutes 35 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1112 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1112/

12 tests failed.
FAILED:  org.apache.solr.cloud.hdfs.HdfsUnloadDistributedZkTest.test

Error Message:
Error from server at http://127.0.0.1:46438: Error CREATEing SolrCore 'test_unload_shard_and_collection_1': Unable to create core [test_unload_shard_and_collection_1] Caused by: Direct buffer memory

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:46438: Error CREATEing SolrCore 'test_unload_shard_and_collection_1': Unable to create core [test_unload_shard_and_collection_1] Caused by: Direct buffer memory
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:608)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.UnloadDistributedZkTest.testUnloadShardAndCollection(UnloadDistributedZkTest.java:129)
	at org.apache.solr.cloud.UnloadDistributedZkTest.test(UnloadDistributedZkTest.java:70)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.core.OpenCloseCoreStressTest.test10Minutes

Error Message:
Captured an uncaught exception in thread: Thread[id=17412, name=Thread-7291, state=RUNNABLE, group=TGRP-OpenCloseCoreStressTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=17412, name=Thread-7291, state=RUNNABLE, group=TGRP-OpenCloseCoreStressTest]
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigAliasReplication

Error Message:
expected:<1> but was:<0>

Stack Trace:
java.lang.AssertionError: expected:<1> but was:<0>
	at __randomizedtesting.SeedInfo.seed([327C025A1D389D1D:C50FEC02DBD032FB]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.handler.TestReplicationHandler.doTestIndexAndConfigAliasReplication(TestReplicationHandler.java:1329)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestStressReplication

Error Message:
[index.20160909000326436, replication.properties, index.properties, index.20160909000333505, snapshot_metadata] expected:<1> but was:<2>

Stack Trace:
java.lang.AssertionError: [index.20160909000326436, replication.properties, index.properties, index.20160909000333505, snapshot_metadata] expected:<1> but was:<2>
	at __randomizedtesting.SeedInfo.seed([327C025A1D389D1D:E9D7029C1810F4AE]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.apache.solr.handler.TestReplicationHandler.checkForSingleIndex(TestReplicationHandler.java:907)
	at org.apache.solr.handler.TestReplicationHandler.doTestStressReplication(TestReplicationHandler.java:874)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testResilienceWithDeleteByQueryOnTarget

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard2

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard2
	at __randomizedtesting.SeedInfo.seed([327C025A1D389D1D:921E90C0CD615858]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testResilienceWithDeleteByQueryOnTarget(CdcrReplicationDistributedZkTest.java:589)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterLeaderChange

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard2

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard2
	at __randomizedtesting.SeedInfo.seed([327C025A1D389D1D:E08C4EB943973B2F]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterLeaderChange(CdcrReplicationDistributedZkTest.java:306)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationStartStop

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard1

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard1
	at __randomizedtesting.SeedInfo.seed([327C025A1D389D1D:B1BFDCC909E01694]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationStartStop(CdcrReplicationDistributedZkTest.java:196)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchAddsWithDelete

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard2

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard2
	at __randomizedtesting.SeedInfo.seed([327C025A1D389D1D:45667BFEB5219C31]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchAddsWithDelete(CdcrReplicationDistributedZkTest.java:526)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:59906/vo/yy

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:59906/vo/yy
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:516)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:47352

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:47352
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:516)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
	at __randomizedtesting.SeedInfo.seed([327C025A1D389D1D]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest:     1) Thread[id=73355, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=73350, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=73340, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=73330, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=73335, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=73320, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=73325, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=73315, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=73345, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=73360, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 
   1) Thread[id=73355, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=73350, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=73340, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=73330, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=73335, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=73320, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=73325, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=73315, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=73345, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=73360, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([327C025A1D389D1D]:0)




Build Log:
[...truncated 11251 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_327C025A1D389D1D-001/init-core-data-001
   [junit4]   2> 93531 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[327C025A1D389D1D]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 93531 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[327C025A1D389D1D]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 93567 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 93574 INFO  (Thread-82) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 93579 INFO  (Thread-82) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 94174 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.ZkTestServer start zk server on port:40962
   [junit4]   2> 94201 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 94456 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 94523 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Exception causing close of session 0x0 due to java.io.IOException: ZooKeeperServer not running
   [junit4]   2> 95865 INFO  (zkCallback-41-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2e3034b5 name:ZooKeeperConnection Watcher:127.0.0.1:40962 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 95865 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 95865 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 95867 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 96424 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 96435 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 96471 INFO  (zkCallback-42-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@234d5125 name:ZooKeeperConnection Watcher:127.0.0.1:40962/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 96471 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 96472 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 96472 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1
   [junit4]   2> 96479 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/shards
   [junit4]   2> 96486 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection
   [junit4]   2> 96497 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection/shards
   [junit4]   2> 96504 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 96511 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 96835 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 96835 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/schema.xml
   [junit4]   2> 96849 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 96849 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 96863 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 96863 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 96872 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 96872 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/protwords.txt
   [junit4]   2> 96880 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 96880 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/currency.xml
   [junit4]   2> 96890 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 96890 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 96899 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 96899 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 96903 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 96904 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 96911 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 96911 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 96917 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 96918 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 96925 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 96939 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 97066 INFO  (zkCallback-43-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5109ce92 name:ZooKeeperConnection Watcher:127.0.0.1:40962 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 97071 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 97071 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 97071 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 97085 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 97091 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 97111 INFO  (zkCallback-44-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@308ccc60 name:ZooKeeperConnection Watcher:127.0.0.1:40962/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 97111 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 97111 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 97112 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf2/solrconfig.xml
   [junit4]   2> 97112 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/solrconfig.xml
   [junit4]   2> 97151 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf2/schema.xml
   [junit4]   2> 97151 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/schema.xml
   [junit4]   2> 97195 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf2/enumsConfig.xml
   [junit4]   2> 97195 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/enumsConfig.xml
   [junit4]   2> 97227 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf2/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 97227 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 97251 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf2/stopwords.txt
   [junit4]   2> 97251 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/stopwords.txt
   [junit4]   2> 97307 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf2/protwords.txt
   [junit4]   2> 97307 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/protwords.txt
   [junit4]   2> 97343 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf2/currency.xml
   [junit4]   2> 97343 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/currency.xml
   [junit4]   2> 97379 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf2/open-exchange-rates.json
   [junit4]   2> 97379 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/open-exchange-rates.json
   [junit4]   2> 97427 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf2/mapping-ISOLatin1Accent.txt
   [junit4]   2> 97427 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/mapping-ISOLatin1Accent.txt
   [junit4]   2> 97519 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf2/old_synonyms.txt
   [junit4]   2> 97519 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/old_synonyms.txt
   [junit4]   2> 97575 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf2/synonyms.txt
   [junit4]   2> 97575 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/synonyms.txt
   [junit4]   2> 97627 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/elevate.xml to /configs/conf2/elevate.xml
   [junit4]   2> 97627 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/elevate.xml
   [junit4]   2> 97800 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_327C025A1D389D1D-001/control-001/cores/collection1
   [junit4]   2> 97809 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 97814 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@639d645b{/,null,AVAILABLE}
   [junit4]   2> 97831 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.e.j.s.ServerConnector Started ServerConnector@7f09d68f{HTTP/1.1,[http/1.1]}{127.0.0.1:52467}
   [junit4]   2> 97831 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.e.j.s.Server Started @105059ms
   [junit4]   2> 97831 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_327C025A1D389D1D-001/tempDir-001/control/data, hostContext=/, hostPort=52467, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_327C025A1D389D1D-001/control-001/cores}
   [junit4]   2> 97831 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 97831 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_327C025A1D389D1D-001/control-001'
   [junit4]   2> 97831 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 97831 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 97832 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 97851 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 97887 INFO  (zkCallback-45-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@71354b85 name:ZooKeeperConnection Watcher:127.0.0.1:40962/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 97887 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 97887 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 97899 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 97899 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_327C025A1D389D1D-001/control-001/solr.xml
   [junit4]   2> 97918 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_327C025A1D389D1D-001/control-001/cores
   [junit4]   2> 97918 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.CoreContainer New CoreContainer 1565325419
   [junit4]   2> 97918 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_327C025A1D389D1D-001/control-001]
   [junit4]   2> 97919 WARN  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.CoreContainer Couldn't add files from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_327C025A1D389D1D-001/control-001/lib to classpath: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_327C025A1D389D1D-001/control-001/lib
   [junit4]   2> 97919 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,
   [junit4]   2> 97929 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.u.UpdateShardHandler Created UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&maxConnections=100000&maxConnectionsPerHost=100000
   [junit4]   2> 97929 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 97929 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 97929 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:40962/solr
   [junit4]   2> 98129 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 98129 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 98130 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 98135 INFO  (zkCallback-48-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@1f7537 name:ZooKeeperConnection Watcher:127.0.0.1:40962 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 98135 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 98136 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 98150 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$108/20832659@798a2508
   [junit4]   2> 98155 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 98175 INFO  (zkCallback-49-thread-1-processing-n:127.0.0.1:52467_) [n:127.0.0.1:52467_    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6a22a12e name:ZooKeeperConnection Watcher:127.0.0.1:40962/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 98175 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 98199 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
   [junit4]   2> 98228 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
   [junit4]   2> 98247 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
   [junit4]   2> 98259 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
   [junit4]   2> 98262 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
   [junit4]   2> 98284 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 98287 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 98289 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 98291 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 98293 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 98304 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.ZkStateReader Loaded empty cluster properties
   [junit4]   2> 98314 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 98366 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
   [junit4]   2> 98368 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
   [junit4]   2> 98371 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 98374 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/96560126554013702-127.0.0.1:52467_-n_0000000000
   [junit4]   2> 98378 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:52467_
   [junit4]   2> 98378 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
   [junit4]   2> 98431 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.Overseer Overseer (id=96560126554013702-127.0.0.1:52467_-n_0000000000) starting
   [junit4]   2> 98503 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
   [junit4]   2> 98790 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=10000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 98800 INFO  (OverseerCollectionConfigSetProcessor-96560126554013702-127.0.0.1:52467_-n_0000000000) [n:127.0.0.1:52467_    ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
   [junit4]   2> 98799 INFO  (OverseerStateUpdate-96560126554013702-127.0.0.1:52467_-n_0000000000) [n:127.0.0.1:52467_    ] o.a.s.c.Overseer Starting to work on the main queue
   [junit4]   2> 98808 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52467_
   [junit4]   2> 98808 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:52467_
   [junit4]   2> 98827 INFO  (zkCallback-49-thread-1-processing-n:127.0.0.1:52467_) [n:127.0.0.1:52467_    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 98826 INFO  (OverseerStateUpdate-96560126554013702-127.0.0.1:52467_-n_0000000000) [n:127.0.0.1:52467_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 98841 INFO  (zkCallback-49-thread-1-processing-n:127.0.0.1:52467_) [n:127.0.0.1:52467_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (1) -> (1)
   [junit4]   2> 99096 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 99096 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 99102 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_327C025A1D389D1D-001/control-001/cores
   [junit4]   2> 99115 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[327C025A1D389D1D]) [n:127.0.0.1:52467_    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.

[...truncated too long message...]

lassicSimilarity, locale=sr-Latn-RS, timezone=America/Virgin
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=228931416,total=522190848
   [junit4]   2> NOTE: All tests run in this JVM: [TestObjectReleaseTracker, StatsComponentTest, MultiThreadedOCPTest, PluginInfoTest, TestMergePolicyConfig, StatelessScriptUpdateProcessorFactoryTest, TestGroupingSearch, SolrCoreCheckLockOnStartupTest, TestSolrXml, TestPostingsSolrHighlighter, SuggesterTSTTest, TestManagedSynonymFilterFactory, TestFreeTextSuggestions, DistributedFacetPivotSmallTest, FullSolrCloudDistribCmdsTest, TestAnalyzeInfixSuggestions, LukeRequestHandlerTest, TestDistributedMissingSort, NoCacheHeaderTest, DeleteStatusTest, TestDistribIDF, CircularListTest, AutoCommitTest, TestLFUCache, SuggesterTest, TestSolrQueryParserResource, BlockJoinFacetDistribTest, AddSchemaFieldsUpdateProcessorFactoryTest, SolrPluginUtilsTest, GraphQueryTest, DirectUpdateHandlerTest, CloneFieldUpdateProcessorFactoryTest, TolerantUpdateProcessorTest, TestSubQueryTransformerCrossCore, QueryParsingTest, DistribJoinFromCollectionTest, FieldAnalysisRequestHandlerTest, BlockDirectoryTest, TestSolrDeletionPolicy1, TestSchemaManager, ZkStateReaderTest, TestUninvertingReader, TestFieldResource, TestFilteredDocIdSet, TestExceedMaxTermLength, TestXIncludeConfig, SpatialHeatmapFacetsTest, EnumFieldTest, TestExtendedDismaxParser, DistributedQueryComponentOptimizationTest, TestSort, JSONWriterTest, TestPhraseSuggestions, InfoHandlerTest, TestComplexPhraseQParserPlugin, TestStressUserVersions, TestTestInjection, CoreAdminHandlerTest, BasicDistributedZk2Test, DocValuesMultiTest, TestOnReconnectListenerSupport, DeleteNodeTest, SharedFSAutoReplicaFailoverTest, CustomCollectionTest, TestLegacyFieldCache, TestCloudManagedSchema, TestTolerantUpdateProcessorCloud, TestConfigSetImmutable, UnloadDistributedZkTest, HdfsCollectionsAPIDistributedZkTest, CoreAdminCreateDiscoverTest, TimeZoneUtilsTest, DOMUtilTest, TestSolrCloudWithSecureImpersonation, HdfsThreadLeakTest, FieldMutatingUpdateProcessorTest, TestSlowCompositeReaderWrapper, TestSearcherReuse, HdfsWriteToMultipleCollectionsTest, ConfigSetsAPITest, RuleEngineTest, TestPerFieldSimilarityWithDefaultOverride, SSLMigrationTest, OpenExchangeRatesOrgProviderTest, SuggesterWFSTTest, SolrCLIZkUtilsTest, PrimitiveFieldTypeTest, TestIntervalFaceting, BitVectorTest, ChangedSchemaMergeTest, ResponseLogComponentTest, TestDynamicFieldCollectionResource, TestRequestStatusCollectionAPI, HdfsLockFactoryTest, TestZkChroot, TestQueryWrapperFilter, TestSchemaVersionResource, DirectoryFactoryTest, TestStressReorder, TestFieldCacheReopen, BadIndexSchemaTest, TestClassicSimilarityFactory, ChaosMonkeySafeLeaderTest, SyncSliceTest, OpenCloseCoreStressTest, OverseerTest, LeaderElectionIntegrationTest, ShardRoutingTest, RecoveryZkTest, TestRandomFaceting, LeaderElectionTest, ZkSolrClientTest, TestDistributedSearch, PreAnalyzedFieldTest, TestSystemIdResolver, PrimUtilsTest, RAMDirectoryFactoryTest, TestSolrJ, TestLRUCache, TestUtils, ZkNodePropsTest, SliceStateTest, FileUtilsTest, TestRTGBase, CursorPagingTest, DistributedIntervalFacetingTest, AsyncMigrateRouteKeyTest, CleanupOldIndexTest, CloudExitableDirectoryReaderTest, CollectionReloadTest, CollectionStateFormat2Test, CollectionsAPIAsyncDistributedZkTest, ConcurrentDeleteAndCreateCollectionTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ConcurrentDeleteAndCreateCollectionTest -Dtests.seed=327C025A1D389D1D -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=sr-Latn-RS -Dtests.timezone=America/Virgin -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J2 | ConcurrentDeleteAndCreateCollectionTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([327C025A1D389D1D]:0)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #2: com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 
   [junit4]    >    1) Thread[id=73355, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    2) Thread[id=73350, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    3) Thread[id=73340, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    4) Thread[id=73330, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    5) Thread[id=73335, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    6) Thread[id=73320, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    7) Thread[id=73325, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    8) Thread[id=73315, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >    9) Thread[id=73345, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    >   10) Thread[id=73360, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
   [junit4]    >         at java.lang.Thread.sleep(Native Method)
   [junit4]    >         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
   [junit4]    >         at java.lang.Thread.run(Thread.java:745)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([327C025A1D389D1D]:0)
   [junit4] Completed [452/631 (7!)] on J2 in 88.80s, 2 tests, 1 failure, 1 error <<< FAILURES!

[...truncated 625 lines...]
   [junit4] JVM J2: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20160909_030821_326.sysout
   [junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid24515.hprof ...
   [junit4] Heap dump file created [487528590 bytes in 2.761 secs]
   [junit4] <<< JVM J2: EOF ----

[...truncated 11040 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:715: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid24515.hprof

Total time: 400 minutes 4 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




Re: [JENKINS] Lucene-Solr-NightlyTests-master - Build # 1111 - Still Failing

Posted by Steve Rowe <sa...@gmail.com>.
Looks like the lucene Jenkins slave needs an increase in the open files limit.

Here’s what I added to /etc/sysctl.conf on my Debian box to do this:

   fs.file-max = 2097152

Uwe, you coordinating the recent puppetization of the lucene slave - should we open an INFRA JIRA to make this change?

--
Steve
www.lucidworks.com

> On Sep 7, 2016, at 6:12 AM, Apache Jenkins Server <je...@builds.apache.org> wrote:
> 
> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1111/
> 
> 10 tests failed.
> FAILED:  org.apache.lucene.index.TestIndexWriterReader.testDuringAddDelete
> 
> Error Message:
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J2/temp/lucene.index.TestIndexWriterReader_61A624ED3323A1DA-001/index-SimpleFSDirectory-001/_24x_Lucene50_1.tim: Too many open files
> 
> Stack Trace:
> java.nio.file.FileSystemException: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J2/temp/lucene.index.TestIndexWriterReader_61A624ED3323A1DA-001/index-SimpleFSDirectory-001/_24x_Lucene50_1.tim: Too many open files
> 	at __randomizedtesting.SeedInfo.seed([61A624ED3323A1DA:FF37536A19E223DA]:0)
> 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
> 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
> 	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:271)
> 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newByteChannel(FilterFileSystemProvider.java:212)
> 	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:240)
> 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newByteChannel(FilterFileSystemProvider.java:212)
> 	at java.nio.file.Files.newByteChannel(Files.java:361)
> 	at java.nio.file.Files.newByteChannel(Files.java:407)
> 	at org.apache.lucene.store.SimpleFSDirectory.openInput(SimpleFSDirectory.java:77)
> 	at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:99)
> 	at org.apache.lucene.codecs.blocktree.BlockTreeTermsReader.<init>(BlockTreeTermsReader.java:153)
> 	at org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.fieldsProducer(Lucene50PostingsFormat.java:445)
> 	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:261)
> 	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:341)
> 	at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:106)
> 	at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:74)
> 	at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:145)
> 	at org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:197)
> 	at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:103)
> 	at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:460)
> 	at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:291)
> 	at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:266)
> 	at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:256)
> 	at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:140)
> 	at org.apache.lucene.index.TestIndexWriterReader.testDuringAddDelete(TestIndexWriterReader.java:870)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:498)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> 
> FAILED:  junit.framework.TestSuite.org.apache.lucene.index.TestIndexWriterReader
> 
> Error Message:
> 5 threads leaked from SUITE scope at org.apache.lucene.index.TestIndexWriterReader:     1) Thread[id=1799, name=Thread-1549, state=BLOCKED, group=TGRP-TestIndexWriterReader]         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)    2) Thread[id=1801, name=Thread-1551, state=RUNNABLE, group=TGRP-TestIndexWriterReader]         at org.apache.lucene.codecs.compressing.CompressingTermVectorsReader.get(CompressingTermVectorsReader.java:417)         at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat$AssertingTermVectorsReader.get(AssertingTermVectorsFormat.java:71)         at org.apache.lucene.codecs.TermVectorsWriter.merge(TermVectorsWriter.java:223)         at org.apache.lucene.index.SegmentMerger.mergeVectors(SegmentMerger.java:210)         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:144)         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4287)         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3864)         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)    3) Thread[id=1802, name=Thread-1552, state=BLOCKED, group=TGRP-TestIndexWriterReader]         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)    4) Thread[id=1800, name=Thread-1550, state=BLOCKED, group=TGRP-TestIndexWriterReader]         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)    5) Thread[id=1798, name=Thread-1548, state=BLOCKED, group=TGRP-TestIndexWriterReader]         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
> 
> Stack Trace:
> com.carrotsearch.randomizedtesting.ThreadLeakError: 5 threads leaked from SUITE scope at org.apache.lucene.index.TestIndexWriterReader: 
>   1) Thread[id=1799, name=Thread-1549, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>        at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
>        at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>        at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>        at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>        at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   2) Thread[id=1801, name=Thread-1551, state=RUNNABLE, group=TGRP-TestIndexWriterReader]
>        at org.apache.lucene.codecs.compressing.CompressingTermVectorsReader.get(CompressingTermVectorsReader.java:417)
>        at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat$AssertingTermVectorsReader.get(AssertingTermVectorsFormat.java:71)
>        at org.apache.lucene.codecs.TermVectorsWriter.merge(TermVectorsWriter.java:223)
>        at org.apache.lucene.index.SegmentMerger.mergeVectors(SegmentMerger.java:210)
>        at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:144)
>        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4287)
>        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3864)
>        at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
>        at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>        at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>        at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>        at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   3) Thread[id=1802, name=Thread-1552, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>        at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
>        at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>        at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>        at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>        at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   4) Thread[id=1800, name=Thread-1550, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>        at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
>        at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>        at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>        at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>        at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   5) Thread[id=1798, name=Thread-1548, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>        at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
>        at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>        at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>        at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>        at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
> 	at __randomizedtesting.SeedInfo.seed([61A624ED3323A1DA]:0)
> 
> 
> FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIDistributedZkTest
> 
> Error Message:
> 1 thread leaked from SUITE scope at org.apache.solr.cloud.CollectionsAPIDistributedZkTest:     1) Thread[id=125227, name=searcherExecutor-10822-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)
> 
> Stack Trace:
> com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.CollectionsAPIDistributedZkTest: 
>   1) Thread[id=125227, name=searcherExecutor-10822-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
>        at sun.misc.Unsafe.park(Native Method)
>        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
>        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
>        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
>        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
>        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
>        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>        at java.lang.Thread.run(Thread.java:745)
> 	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32]:0)
> 
> 
> FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIDistributedZkTest
> 
> Error Message:
> There are still zombie threads that couldn't be terminated:    1) Thread[id=125227, name=searcherExecutor-10822-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)
> 
> Stack Trace:
> com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
>   1) Thread[id=125227, name=searcherExecutor-10822-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
>        at sun.misc.Unsafe.park(Native Method)
>        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
>        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
>        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
>        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
>        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
>        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
>        at java.lang.Thread.run(Thread.java:745)
> 	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32]:0)
> 
> 
> FAILED:  org.apache.solr.cloud.hdfs.StressHdfsTest.test
> 
> Error Message:
> Could not find collection:delete_data_dir
> 
> Stack Trace:
> java.lang.AssertionError: Could not find collection:delete_data_dir
> 	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32:EE8AB604096653CA]:0)
> 	at org.junit.Assert.fail(Assert.java:93)
> 	at org.junit.Assert.assertTrue(Assert.java:43)
> 	at org.junit.Assert.assertNotNull(Assert.java:526)
> 	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:154)
> 	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:139)
> 	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:134)
> 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:856)
> 	at org.apache.solr.cloud.hdfs.StressHdfsTest.createAndDeleteCollection(StressHdfsTest.java:159)
> 	at org.apache.solr.cloud.hdfs.StressHdfsTest.test(StressHdfsTest.java:103)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:498)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
> 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
> 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> 
> FAILED:  org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.test
> 
> Error Message:
> Timeout occured while waiting response from server at: https://127.0.0.1:39500/vum
> 
> Stack Trace:
> org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:39500/vum
> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
> 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
> 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:516)
> 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:498)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
> 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
> 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
> 	at java.lang.Thread.run(Thread.java:745)
> Caused by: java.net.SocketTimeoutException: Read timed out
> 	at java.net.SocketInputStream.socketRead0(Native Method)
> 	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
> 	at java.net.SocketInputStream.read(SocketInputStream.java:170)
> 	at java.net.SocketInputStream.read(SocketInputStream.java:141)
> 	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
> 	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
> 	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
> 	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
> 	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
> 	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
> 	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
> 	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
> 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
> 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
> 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
> 	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
> 	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
> 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
> 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
> 	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
> 	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
> 	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
> 	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
> 	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
> 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
> 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
> 	... 47 more
> 
> 
> FAILED:  org.apache.solr.handler.component.DistributedFacetPivotLongTailTest.test
> 
> Error Message:
> IOException occured when talking to server at: https://127.0.0.1:46729/_/d/collection1
> 
> Stack Trace:
> org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: https://127.0.0.1:46729/_/d/collection1
> 	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32:EE8AB604096653CA]:0)
> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:622)
> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
> 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
> 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
> 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
> 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:152)
> 	at org.apache.solr.handler.component.DistributedFacetPivotLongTailTest.test(DistributedFacetPivotLongTailTest.java:71)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:498)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
> 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
> 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
> 	at java.lang.Thread.run(Thread.java:745)
> Caused by: org.apache.http.conn.ConnectTimeoutException: Connect to 127.0.0.1:46729 [/127.0.0.1] failed: Read timed out
> 	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:143)
> 	at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
> 	at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
> 	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
> 	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
> 	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
> 	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
> 	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
> 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
> 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
> 	... 48 more
> Caused by: java.net.SocketTimeoutException: Read timed out
> 	at java.net.SocketInputStream.socketRead0(Native Method)
> 	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
> 	at java.net.SocketInputStream.read(SocketInputStream.java:170)
> 	at java.net.SocketInputStream.read(SocketInputStream.java:141)
> 	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
> 	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
> 	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
> 	at sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1375)
> 	at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1403)
> 	at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1387)
> 	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.createLayeredSocket(SSLConnectionSocketFactory.java:394)
> 	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:353)
> 	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:134)
> 	... 58 more
> 
> 
> FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test
> 
> Error Message:
> Timeout occured while waiting response from server at: https://127.0.0.1:35113
> 
> Stack Trace:
> org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:35113
> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
> 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
> 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
> 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:516)
> 	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> 	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:498)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
> 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
> 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
> 	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
> 	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
> 	at java.lang.Thread.run(Thread.java:745)
> Caused by: java.net.SocketTimeoutException: Read timed out
> 	at java.net.SocketInputStream.socketRead0(Native Method)
> 	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
> 	at java.net.SocketInputStream.read(SocketInputStream.java:170)
> 	at java.net.SocketInputStream.read(SocketInputStream.java:141)
> 	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
> 	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
> 	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
> 	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
> 	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
> 	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
> 	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
> 	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
> 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
> 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
> 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
> 	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
> 	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
> 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
> 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
> 	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
> 	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
> 	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
> 	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
> 	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
> 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
> 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
> 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
> 	... 47 more
> 
> 
> FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest
> 
> Error Message:
> ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
> 
> Stack Trace:
> java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
> 	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32]:0)
> 	at org.junit.Assert.fail(Assert.java:93)
> 	at org.junit.Assert.assertTrue(Assert.java:43)
> 	at org.junit.Assert.assertNull(Assert.java:551)
> 	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
> 	at sun.reflect.GeneratedMethodAccessor73.invoke(Unknown Source)
> 	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> 	at java.lang.reflect.Method.invoke(Method.java:498)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
> 	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
> 	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> 	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
> 	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
> 	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
> 	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
> 	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
> 	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
> 	at java.lang.Thread.run(Thread.java:745)
> 
> 
> FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest
> 
> Error Message:
> 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest:     1) Thread[id=28289, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=28294, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=28264, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=28259, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=28269, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=28304, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=28284, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=28299, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=28274, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=28279, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)
> 
> Stack Trace:
> com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 
>   1) Thread[id=28289, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
>        at java.lang.Thread.sleep(Native Method)
>        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
>        at java.lang.Thread.run(Thread.java:745)
>   2) Thread[id=28294, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
>        at java.lang.Thread.sleep(Native Method)
>        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
>        at java.lang.Thread.run(Thread.java:745)
>   3) Thread[id=28264, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
>        at java.lang.Thread.sleep(Native Method)
>        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
>        at java.lang.Thread.run(Thread.java:745)
>   4) Thread[id=28259, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
>        at java.lang.Thread.sleep(Native Method)
>        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
>        at java.lang.Thread.run(Thread.java:745)
>   5) Thread[id=28269, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
>        at java.lang.Thread.sleep(Native Method)
>        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
>        at java.lang.Thread.run(Thread.java:745)
>   6) Thread[id=28304, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
>        at java.lang.Thread.sleep(Native Method)
>        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
>        at java.lang.Thread.run(Thread.java:745)
>   7) Thread[id=28284, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
>        at java.lang.Thread.sleep(Native Method)
>        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
>        at java.lang.Thread.run(Thread.java:745)
>   8) Thread[id=28299, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
>        at java.lang.Thread.sleep(Native Method)
>        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
>        at java.lang.Thread.run(Thread.java:745)
>   9) Thread[id=28274, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
>        at java.lang.Thread.sleep(Native Method)
>        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
>        at java.lang.Thread.run(Thread.java:745)
>  10) Thread[id=28279, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
>        at java.lang.Thread.sleep(Native Method)
>        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
>        at java.lang.Thread.run(Thread.java:745)
> 	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32]:0)
> 
> 
> 
> 
> Build Log:
> [...truncated 913 lines...]
>   [junit4] Suite: org.apache.lucene.index.TestIndexWriterReader
>   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestIndexWriterReader -Dtests.method=testDuringAddDelete -Dtests.seed=61A624ED3323A1DA -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=mk -Dtests.timezone=Europe/Skopje -Dtests.asserts=true -Dtests.file.encoding=UTF-8
>   [junit4] ERROR    142s J2 | TestIndexWriterReader.testDuringAddDelete <<<
>   [junit4]    > Throwable #1: java.nio.file.FileSystemException: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J2/temp/lucene.index.TestIndexWriterReader_61A624ED3323A1DA-001/index-SimpleFSDirectory-001/_24x_Lucene50_1.tim: Too many open files
>   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([61A624ED3323A1DA:FF37536A19E223DA]:0)
>   [junit4]    > 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
>   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
>   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:271)
>   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newByteChannel(FilterFileSystemProvider.java:212)
>   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:240)
>   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newByteChannel(FilterFileSystemProvider.java:212)
>   [junit4]    > 	at java.nio.file.Files.newByteChannel(Files.java:361)
>   [junit4]    > 	at java.nio.file.Files.newByteChannel(Files.java:407)
>   [junit4]    > 	at org.apache.lucene.store.SimpleFSDirectory.openInput(SimpleFSDirectory.java:77)
>   [junit4]    > 	at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:99)
>   [junit4]    > 	at org.apache.lucene.codecs.blocktree.BlockTreeTermsReader.<init>(BlockTreeTermsReader.java:153)
>   [junit4]    > 	at org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.fieldsProducer(Lucene50PostingsFormat.java:445)
>   [junit4]    > 	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:261)
>   [junit4]    > 	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:341)
>   [junit4]    > 	at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:106)
>   [junit4]    > 	at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:74)
>   [junit4]    > 	at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:145)
>   [junit4]    > 	at org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:197)
>   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:103)
>   [junit4]    > 	at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:460)
>   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:291)
>   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:266)
>   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:256)
>   [junit4]    > 	at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:140)
>   [junit4]    > 	at org.apache.lucene.index.TestIndexWriterReader.testDuringAddDelete(TestIndexWriterReader.java:870)
>   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
>   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J2/temp/lucene.index.TestIndexWriterReader_61A624ED3323A1DA-001
>   [junit4]   2> Sep 07, 2016 4:42:35 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
>   [junit4]   2> WARNING: Will linger awaiting termination of 5 leaked thread(s).
>   [junit4]   2> Sep 07, 2016 4:42:55 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
>   [junit4]   2> SEVERE: 5 threads leaked from SUITE scope at org.apache.lucene.index.TestIndexWriterReader: 
>   [junit4]   2>    1) Thread[id=1799, name=Thread-1549, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   [junit4]   2>    2) Thread[id=1801, name=Thread-1551, state=RUNNABLE, group=TGRP-TestIndexWriterReader]
>   [junit4]   2>         at org.apache.lucene.codecs.compressing.CompressingTermVectorsReader.get(CompressingTermVectorsReader.java:417)
>   [junit4]   2>         at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat$AssertingTermVectorsReader.get(AssertingTermVectorsFormat.java:71)
>   [junit4]   2>         at org.apache.lucene.codecs.TermVectorsWriter.merge(TermVectorsWriter.java:223)
>   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.mergeVectors(SegmentMerger.java:210)
>   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:144)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4287)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3864)
>   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   [junit4]   2>    3) Thread[id=1802, name=Thread-1552, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   [junit4]   2>    4) Thread[id=1800, name=Thread-1550, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   [junit4]   2>    5) Thread[id=1798, name=Thread-1548, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   [junit4]   2> Sep 07, 2016 4:42:55 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
>   [junit4]   2> INFO: Starting to interrupt leaked threads:
>   [junit4]   2>    1) Thread[id=1799, name=Thread-1549, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>   [junit4]   2>    2) Thread[id=1801, name=Thread-1551, state=RUNNABLE, group=TGRP-TestIndexWriterReader]
>   [junit4]   2>    3) Thread[id=1802, name=Thread-1552, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>   [junit4]   2>    4) Thread[id=1800, name=Thread-1550, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>   [junit4]   2>    5) Thread[id=1798, name=Thread-1548, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>   [junit4]   2> Sep 07, 2016 4:42:55 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
>   [junit4]   2> INFO: All leaked threads terminated.
>   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {field1=Lucene50(blocksize=128), field=Lucene50(blocksize=128), indexname=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), foo=Lucene50(blocksize=128), id=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), field3=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), field2=FSTOrd50, field5=Lucene50(blocksize=128), field4=Lucene50(blocksize=128)}, docValues:{}, maxPointsInLeafNode=1314, maxMBSortInHeap=5.621346163809926, sim=RandomSimilarity(queryNorm=true): {field1=DFR I(F)B1, field=DFR I(n)L2, indexname=DFR GLZ(0.3), foo=DFR GLZ(0.3), field3=DFR I(ne)Z(0.3), field2=IB SPL-D1, field5=DFR I(F)1, field4=DFR I(F)LZ(0.3)}, locale=mk, timezone=Europe/Skopje
>   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=139897912,total=268959744
>   [junit4]   2> NOTE: All tests run in this JVM: [Test2BPagedBytes, TestStressIndexing, TestGeoUtils, TestNoMergePolicy, TestRollback, TestBlockPostingsFormat, TestTermQuery, TestTopDocsMerge, TestFilterIterator, TestStressDeletes, TestIntsRef, TestDocBoost, TestNamedSPILoader, TestTransactions, TestRollingBuffer, TestFilterWeight, TestIndexWriterThreadsToSegments, TestLogMergePolicy, TestHighCompressionMode, TestDocValuesIndexing, TestRegexpRandom2, TestSingleInstanceLockFactory, TestLucene62SegmentInfoFormat, TestSegmentMerger, TestIndexWriterCommit, TestNRTCachingDirectory, TestAllFilesHaveCodecHeader, TestFuzzyQuery, Test2BPostings, TestDocumentsWriterDeleteQueue, TestByteBlockPool, TestScorerPerf, TestPayloadsOnVectors, TestParallelLeafReader, TestSegmentTermDocs, TestParallelTermEnum, TestDocument, TestVersion, TestAtomicUpdate, TestConjunctionDISI, TestBagOfPostings, TestMultiTermConstantScore, TestSnapshotDeletionPolicy, TestMmapDirectory, TestSpansEnum, TestSpanNotQuery, TestTimSorter, TestWildcardRandom, TestInPlaceMergeSorter, TestFixedBitDocIdSet, TestSimilarity, TestSearchWithThreads, TestCharacterUtils, TestUpgradeIndexMergePolicy, TestBlendedTermQuery, TestBooleanQuery, TestArrayUtil, TestSimpleSearchEquivalence, TestSpanMultiTermQueryWrapper, TestDeletionPolicy, TestMultiPhraseEnum, TestLazyProxSkipping, TestForUtil, TestSpanExplanations, TestReaderWrapperDVTypeCheck, TestFastCompressionMode, TestRegexpQuery, TestTopFieldCollector, TestMSBRadixSorter, TestIndexWriterMerging, TestFileSwitchDirectory, TestIndexWriter, TestIndexWriterReader]
>   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestIndexWriterReader -Dtests.seed=61A624ED3323A1DA -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=mk -Dtests.timezone=Europe/Skopje -Dtests.asserts=true -Dtests.file.encoding=UTF-8
>   [junit4] ERROR   0.00s J2 | TestIndexWriterReader (suite) <<<
>   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 5 threads leaked from SUITE scope at org.apache.lucene.index.TestIndexWriterReader: 
>   [junit4]    >    1) Thread[id=1799, name=Thread-1549, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>   [junit4]    >         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>   [junit4]    >         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>   [junit4]    >         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   [junit4]    >    2) Thread[id=1801, name=Thread-1551, state=RUNNABLE, group=TGRP-TestIndexWriterReader]
>   [junit4]    >         at org.apache.lucene.codecs.compressing.CompressingTermVectorsReader.get(CompressingTermVectorsReader.java:417)
>   [junit4]    >         at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat$AssertingTermVectorsReader.get(AssertingTermVectorsFormat.java:71)
>   [junit4]    >         at org.apache.lucene.codecs.TermVectorsWriter.merge(TermVectorsWriter.java:223)
>   [junit4]    >         at org.apache.lucene.index.SegmentMerger.mergeVectors(SegmentMerger.java:210)
>   [junit4]    >         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:144)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4287)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3864)
>   [junit4]    >         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>   [junit4]    >         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>   [junit4]    >         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   [junit4]    >    3) Thread[id=1802, name=Thread-1552, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>   [junit4]    >         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>   [junit4]    >         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>   [junit4]    >         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   [junit4]    >    4) Thread[id=1800, name=Thread-1550, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>   [junit4]    >         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>   [junit4]    >         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>   [junit4]    >         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   [junit4]    >    5) Thread[id=1798, name=Thread-1548, state=BLOCKED, group=TGRP-TestIndexWriterReader]
>   [junit4]    >         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
>   [junit4]    >         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
>   [junit4]    >         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
>   [junit4]    >         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
>   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([61A624ED3323A1DA]:0)
>   [junit4] Completed [200/432 (1!)] on J2 in 252.56s, 23 tests, 2 errors <<< FAILURES!
> 
> [...truncated 10658 lines...]
>   [junit4] Suite: org.apache.solr.cloud.hdfs.StressHdfsTest
>   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.StressHdfsTest_66DE89DEA79A3E32-001/init-core-data-001
>   [junit4]   2> 814899 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
>   [junit4]   2> 814899 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /_a/m
>   [junit4]   1> Formatting using clusterid: testClusterID
>   [junit4]   2> 815026 WARN  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
>   [junit4]   2> 815069 WARN  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
>   [junit4]   2> 815081 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log jetty-6.1.26
>   [junit4]   2> 815125 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_33702_hdfs____13ay83/webapp
>   [junit4]   2> 816226 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:33702
>   [junit4]   2> 817177 WARN  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
>   [junit4]   2> 817179 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log jetty-6.1.26
>   [junit4]   2> 817200 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_46024_datanode____.2mcfop/webapp
>   [junit4]   2> 818238 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:46024
>   [junit4]   2> 818469 WARN  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
>   [junit4]   2> 818470 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log jetty-6.1.26
>   [junit4]   2> 818502 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_55265_datanode____.v9bv0o/webapp
>   [junit4]   2> 818789 INFO  (IPC Server handler 6 on 59517) [    ] BlockStateChange BLOCK* processReport: from storage DS-124aedd7-53e1-4abe-9593-1749020619f8 node DatanodeRegistration(127.0.0.1:39806, datanodeUuid=44048ab0-2317-4266-b18a-947e4bc5cd29, infoPort=51796, infoSecurePort=0, ipcPort=39160, storageInfo=lv=-56;cid=testClusterID;nsid=2101821981;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
>   [junit4]   2> 818789 INFO  (IPC Server handler 6 on 59517) [    ] BlockStateChange BLOCK* processReport: from storage DS-0427d4bd-1eeb-4af5-a408-0b312310f3bb node DatanodeRegistration(127.0.0.1:39806, datanodeUuid=44048ab0-2317-4266-b18a-947e4bc5cd29, infoPort=51796, infoSecurePort=0, ipcPort=39160, storageInfo=lv=-56;cid=testClusterID;nsid=2101821981;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
>   [junit4]   2> 819366 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:55265
>   [junit4]   2> 819712 INFO  (IPC Server handler 3 on 59517) [    ] BlockStateChange BLOCK* processReport: from storage DS-0a413b87-a6df-47f1-b731-f3928bdb7bd0 node DatanodeRegistration(127.0.0.1:38537, datanodeUuid=6109fe6a-418b-4810-8965-110624cd978c, infoPort=48162, infoSecurePort=0, ipcPort=33212, storageInfo=lv=-56;cid=testClusterID;nsid=2101821981;c=0), blocks: 0, hasStaleStorage: true, processing time: 86 msecs
>   [junit4]   2> 819712 INFO  (IPC Server handler 3 on 59517) [    ] BlockStateChange BLOCK* processReport: from storage DS-394f120c-9225-45db-9d4d-a79e0357d0a0 node DatanodeRegistration(127.0.0.1:38537, datanodeUuid=6109fe6a-418b-4810-8965-110624cd978c, infoPort=48162, infoSecurePort=0, ipcPort=33212, storageInfo=lv=-56;cid=testClusterID;nsid=2101821981;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
>   [junit4]   2> 819976 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
>   [junit4]   2> 819977 INFO  (Thread-1303) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
>   [junit4]   2> 819977 INFO  (Thread-1303) [    ] o.a.s.c.ZkTestServer Starting server
>   [junit4]   2> 820180 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.ZkTestServer start zk server on port:43044
>   [junit4]   2> 820180 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
>   [junit4]   2> 820189 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
>   [junit4]   2> 820201 INFO  (zkCallback-698-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6ea7009c name:ZooKeeperConnection Watcher:127.0.0.1:43044 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 820201 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>   [junit4]   2> 820201 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
>   [junit4]   2> 820201 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
>   [junit4]   2> 820210 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
>   [junit4]   2> 820215 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
>   [junit4]   2> 820239 INFO  (zkCallback-699-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@1095833c name:ZooKeeperConnection Watcher:127.0.0.1:43044/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
>   [junit4]   2> 820241 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
>   [junit4]   2> 820241 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
>   [junit4]   2> 820241 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1
>   [junit4]   2> 820242 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/shards
>   [junit4]   2> 820244 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection
>   [junit4]   2> 820244 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection/shards
>   [junit4]   2> 820250 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
>   [junit4]   2> 820250 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.xml
>   [junit4]   2> 820252 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
>   [junit4]   2> 820252 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/schema.xml
>   [junit4]   2> 820253 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>   [junit4]   2> 820253 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
>   [junit4]   2> 820255 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
>   [junit4]   2> 820255 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/stopwords.txt
>   [junit4]   2> 820262 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
>   [junit4]   2> 820262 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/protwords.txt
>   [junit4]   2> 820263 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
>   [junit4]   2> 820263 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/currency.xml
>   [junit4]   2> 820264 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
>   [junit4]   2> 820264 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/enumsConfig.xml
>   [junit4]   2> 820270 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenki
> 
> [...truncated too long message...]
> 
> ctory Closing directory, CoreContainer#isShutdown=true
>   [junit4]   2> 4696298 INFO  (coreCloseExecutor-5736-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory Closing directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedFacetPivotLongTailTest_66DE89DEA79A3E32-001/tempDir-001/shard2/cores/collection1/data/index
>   [junit4]   2> 4696305 INFO  (TEST-DistributedFacetPivotLongTailTest.test-seed#[66DE89DEA79A3E32]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@17e66fe8{/_/d,null,UNAVAILABLE}
>   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=DistributedFacetPivotLongTailTest -Dtests.method=test -Dtests.seed=66DE89DEA79A3E32 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=cs -Dtests.timezone=Etc/GMT-6 -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
>   [junit4] ERROR    131s J0 | DistributedFacetPivotLongTailTest.test <<<
>   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: https://127.0.0.1:46729/_/d/collection1
>   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32:EE8AB604096653CA]:0)
>   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:622)
>   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
>   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
>   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
>   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
>   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
>   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:152)
>   [junit4]    > 	at org.apache.solr.handler.component.DistributedFacetPivotLongTailTest.test(DistributedFacetPivotLongTailTest.java:71)
>   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
>   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
>   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
>   [junit4]    > Caused by: org.apache.http.conn.ConnectTimeoutException: Connect to 127.0.0.1:46729 [/127.0.0.1] failed: Read timed out
>   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:143)
>   [junit4]    > 	at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
>   [junit4]    > 	at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
>   [junit4]    > 	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
>   [junit4]    > 	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
>   [junit4]    > 	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
>   [junit4]    > 	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
>   [junit4]    > 	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
>   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
>   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
>   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
>   [junit4]    > 	... 48 more
>   [junit4]    > Caused by: java.net.SocketTimeoutException: Read timed out
>   [junit4]    > 	at java.net.SocketInputStream.socketRead0(Native Method)
>   [junit4]    > 	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
>   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:170)
>   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:141)
>   [junit4]    > 	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
>   [junit4]    > 	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
>   [junit4]    > 	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
>   [junit4]    > 	at sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1375)
>   [junit4]    > 	at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1403)
>   [junit4]    > 	at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1387)
>   [junit4]    > 	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.createLayeredSocket(SSLConnectionSocketFactory.java:394)
>   [junit4]    > 	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:353)
>   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:134)
>   [junit4]    > 	... 58 more
>   [junit4]   2> 4696316 INFO  (SUITE-DistributedFacetPivotLongTailTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
>   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedFacetPivotLongTailTest_66DE89DEA79A3E32-001
>   [junit4]   2> NOTE: test params are: codec=Lucene62, sim=ClassicSimilarity, locale=cs, timezone=Etc/GMT-6
>   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=251613816,total=477626368
>   [junit4]   2> NOTE: All tests run in this JVM: [OverseerModifyCollectionTest, RequiredFieldsTest, DistributedDebugComponentTest, TestMissingGroups, TestIBSimilarityFactory, CdcrBootstrapTest, TestManagedSynonymFilterFactory, LeaderInitiatedRecoveryOnShardRestartTest, TestSubQueryTransformerCrossCore, TestOnReconnectListenerSupport, ZkStateReaderTest, TestSolrQueryParserResource, ResourceLoaderTest, TestConfigSetImmutable, TestSolr4Spatial, TestFreeTextSuggestions, TestReplicationHandlerBackup, ReturnFieldsTest, TestClassicSimilarityFactory, HdfsThreadLeakTest, SharedFSAutoReplicaFailoverUtilsTest, TestCSVLoader, LeaderElectionIntegrationTest, ZkNodePropsTest, TestRemoteStreaming, TestSchemaManager, EnumFieldTest, BasicZkTest, DirectoryFactoryTest, RemoteQueryErrorTest, SOLR749Test, TestLMDirichletSimilarityFactory, TestStressUserVersions, TestFieldCacheSortRandom, TestDistribDocBasedVersion, ChangedSchemaMergeTest, RuleEngineTest, TestPhraseSuggestions, TestCloudPseudoReturnFields, TestReplicationHandler, DisMaxRequestHandlerTest, TestLegacyFieldCache, TestComplexPhraseQParserPlugin, BasicDistributedZkTest, TestLRUCache, TestDistributedSearch, RestartWhileUpdatingTest, TestMergePolicyConfig, TestRebalanceLeaders, TestInitParams, CacheHeaderTest, BitVectorTest, TestArbitraryIndexDir, TestCloudDeleteByQuery, PathHierarchyTokenizerFactoryTest, TestSchemaNameResource, TestOrdValues, DistributedFacetPivotLargeTest, BlockJoinFacetDistribTest, TestFaceting, SynonymTokenizerTest, SpatialHeatmapFacetsTest, TestMiniSolrCloudClusterBase, OutOfBoxZkACLAndCredentialsProvidersTest, QueryEqualityTest, IndexSchemaTest, TestBadConfig, TestAddFieldRealTimeGet, BlockDirectoryTest, DefaultValueUpdateProcessorTest, TestCollationFieldDocValues, TestFoldingMultitermQuery, TestBlobHandler, SolrCloudExampleTest, DistributedSpellCheckComponentTest, ConnectionReuseTest, PingRequestHandlerTest, DocumentAnalysisRequestHandlerTest, AddSchemaFieldsUpdateProcessorFactoryTest, HdfsCollectionsAPIDistributedZkTest, TestRTGBase, CloneFieldUpdateProcessorFactoryTest, TestSizeLimitedDistributedMap, TestLuceneMatchVersion, TestRawResponseWriter, SignatureUpdateProcessorFactoryTest, TestXmlQParserPlugin, TestComponentsName, TestSQLHandler, HdfsLockFactoryTest, TestIntervalFaceting, TestCloudManagedSchema, TestDynamicFieldCollectionResource, TestSortingResponseWriter, TestHighlightDedupGrouping, UUIDFieldTest, ConnectionManagerTest, AnalysisAfterCoreReloadTest, TestSchemaVersionResource, JSONWriterTest, TestUninvertingReader, TermVectorComponentDistributedTest, TestGroupingSearch, SimpleCollectionCreateDeleteTest, DistributedFacetPivotLongTailTest]
>   [junit4] Completed [632/633 (5!)] on J0 in 130.67s, 1 test, 1 error <<< FAILURES!
> 
> [...truncated 1 lines...]
>   [junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20160907_083928_132.sysout
>   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
>   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
>   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid30315.hprof ...
>   [junit4] Heap dump file created [707246725 bytes in 19.813 secs]
>   [junit4] <<< JVM J0: EOF ----
> 
> [...truncated 11039 lines...]
> BUILD FAILED
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
> /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:715: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
> * java_pid30315.hprof
> 
> Total time: 367 minutes 35 seconds
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> Recording test results
> Email was triggered for: Failure - Any
> Sending email for trigger: Failure - Any
> 
> 
> 
> 
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1111 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1111/

10 tests failed.
FAILED:  org.apache.lucene.index.TestIndexWriterReader.testDuringAddDelete

Error Message:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J2/temp/lucene.index.TestIndexWriterReader_61A624ED3323A1DA-001/index-SimpleFSDirectory-001/_24x_Lucene50_1.tim: Too many open files

Stack Trace:
java.nio.file.FileSystemException: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J2/temp/lucene.index.TestIndexWriterReader_61A624ED3323A1DA-001/index-SimpleFSDirectory-001/_24x_Lucene50_1.tim: Too many open files
	at __randomizedtesting.SeedInfo.seed([61A624ED3323A1DA:FF37536A19E223DA]:0)
	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:271)
	at org.apache.lucene.mockfile.FilterFileSystemProvider.newByteChannel(FilterFileSystemProvider.java:212)
	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:240)
	at org.apache.lucene.mockfile.FilterFileSystemProvider.newByteChannel(FilterFileSystemProvider.java:212)
	at java.nio.file.Files.newByteChannel(Files.java:361)
	at java.nio.file.Files.newByteChannel(Files.java:407)
	at org.apache.lucene.store.SimpleFSDirectory.openInput(SimpleFSDirectory.java:77)
	at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:99)
	at org.apache.lucene.codecs.blocktree.BlockTreeTermsReader.<init>(BlockTreeTermsReader.java:153)
	at org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.fieldsProducer(Lucene50PostingsFormat.java:445)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:261)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:341)
	at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:106)
	at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:74)
	at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:145)
	at org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:197)
	at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:103)
	at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:460)
	at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:291)
	at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:266)
	at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:256)
	at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:140)
	at org.apache.lucene.index.TestIndexWriterReader.testDuringAddDelete(TestIndexWriterReader.java:870)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.lucene.index.TestIndexWriterReader

Error Message:
5 threads leaked from SUITE scope at org.apache.lucene.index.TestIndexWriterReader:     1) Thread[id=1799, name=Thread-1549, state=BLOCKED, group=TGRP-TestIndexWriterReader]         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)    2) Thread[id=1801, name=Thread-1551, state=RUNNABLE, group=TGRP-TestIndexWriterReader]         at org.apache.lucene.codecs.compressing.CompressingTermVectorsReader.get(CompressingTermVectorsReader.java:417)         at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat$AssertingTermVectorsReader.get(AssertingTermVectorsFormat.java:71)         at org.apache.lucene.codecs.TermVectorsWriter.merge(TermVectorsWriter.java:223)         at org.apache.lucene.index.SegmentMerger.mergeVectors(SegmentMerger.java:210)         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:144)         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4287)         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3864)         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)    3) Thread[id=1802, name=Thread-1552, state=BLOCKED, group=TGRP-TestIndexWriterReader]         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)    4) Thread[id=1800, name=Thread-1550, state=BLOCKED, group=TGRP-TestIndexWriterReader]         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)    5) Thread[id=1798, name=Thread-1548, state=BLOCKED, group=TGRP-TestIndexWriterReader]         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 5 threads leaked from SUITE scope at org.apache.lucene.index.TestIndexWriterReader: 
   1) Thread[id=1799, name=Thread-1549, state=BLOCKED, group=TGRP-TestIndexWriterReader]
        at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
        at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
        at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
        at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
        at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   2) Thread[id=1801, name=Thread-1551, state=RUNNABLE, group=TGRP-TestIndexWriterReader]
        at org.apache.lucene.codecs.compressing.CompressingTermVectorsReader.get(CompressingTermVectorsReader.java:417)
        at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat$AssertingTermVectorsReader.get(AssertingTermVectorsFormat.java:71)
        at org.apache.lucene.codecs.TermVectorsWriter.merge(TermVectorsWriter.java:223)
        at org.apache.lucene.index.SegmentMerger.mergeVectors(SegmentMerger.java:210)
        at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:144)
        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4287)
        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3864)
        at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
        at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
        at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
        at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
        at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   3) Thread[id=1802, name=Thread-1552, state=BLOCKED, group=TGRP-TestIndexWriterReader]
        at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
        at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
        at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
        at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
        at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   4) Thread[id=1800, name=Thread-1550, state=BLOCKED, group=TGRP-TestIndexWriterReader]
        at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
        at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
        at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
        at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
        at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   5) Thread[id=1798, name=Thread-1548, state=BLOCKED, group=TGRP-TestIndexWriterReader]
        at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
        at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
        at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
        at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
        at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
	at __randomizedtesting.SeedInfo.seed([61A624ED3323A1DA]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIDistributedZkTest

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.CollectionsAPIDistributedZkTest:     1) Thread[id=125227, name=searcherExecutor-10822-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.CollectionsAPIDistributedZkTest: 
   1) Thread[id=125227, name=searcherExecutor-10822-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.CollectionsAPIDistributedZkTest

Error Message:
There are still zombie threads that couldn't be terminated:    1) Thread[id=125227, name=searcherExecutor-10822-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: There are still zombie threads that couldn't be terminated:
   1) Thread[id=125227, name=searcherExecutor-10822-thread-1, state=WAITING, group=TGRP-CollectionsAPIDistributedZkTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1067)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1127)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32]:0)


FAILED:  org.apache.solr.cloud.hdfs.StressHdfsTest.test

Error Message:
Could not find collection:delete_data_dir

Stack Trace:
java.lang.AssertionError: Could not find collection:delete_data_dir
	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32:EE8AB604096653CA]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:154)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:139)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:134)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:856)
	at org.apache.solr.cloud.hdfs.StressHdfsTest.createAndDeleteCollection(StressHdfsTest.java:159)
	at org.apache.solr.cloud.hdfs.StressHdfsTest.test(StressHdfsTest.java:103)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:39500/vum

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:39500/vum
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:516)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  org.apache.solr.handler.component.DistributedFacetPivotLongTailTest.test

Error Message:
IOException occured when talking to server at: https://127.0.0.1:46729/_/d/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: https://127.0.0.1:46729/_/d/collection1
	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32:EE8AB604096653CA]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:622)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:152)
	at org.apache.solr.handler.component.DistributedFacetPivotLongTailTest.test(DistributedFacetPivotLongTailTest.java:71)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.conn.ConnectTimeoutException: Connect to 127.0.0.1:46729 [/127.0.0.1] failed: Read timed out
	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:143)
	at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
	at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 48 more
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1375)
	at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1403)
	at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1387)
	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.createLayeredSocket(SSLConnectionSocketFactory.java:394)
	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:353)
	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:134)
	... 58 more


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:35113

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:35113
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:516)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.GeneratedMethodAccessor73.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest:     1) Thread[id=28289, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=28294, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=28264, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=28259, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=28269, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=28304, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=28284, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=28299, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=28274, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=28279, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 
   1) Thread[id=28289, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=28294, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=28264, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=28259, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=28269, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=28304, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=28284, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=28299, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=28274, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=28279, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32]:0)




Build Log:
[...truncated 913 lines...]
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterReader
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestIndexWriterReader -Dtests.method=testDuringAddDelete -Dtests.seed=61A624ED3323A1DA -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=mk -Dtests.timezone=Europe/Skopje -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR    142s J2 | TestIndexWriterReader.testDuringAddDelete <<<
   [junit4]    > Throwable #1: java.nio.file.FileSystemException: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J2/temp/lucene.index.TestIndexWriterReader_61A624ED3323A1DA-001/index-SimpleFSDirectory-001/_24x_Lucene50_1.tim: Too many open files
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([61A624ED3323A1DA:FF37536A19E223DA]:0)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleLimitFS.onOpen(HandleLimitFS.java:48)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:81)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:271)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newByteChannel(FilterFileSystemProvider.java:212)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newByteChannel(HandleTrackingFS.java:240)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newByteChannel(FilterFileSystemProvider.java:212)
   [junit4]    > 	at java.nio.file.Files.newByteChannel(Files.java:361)
   [junit4]    > 	at java.nio.file.Files.newByteChannel(Files.java:407)
   [junit4]    > 	at org.apache.lucene.store.SimpleFSDirectory.openInput(SimpleFSDirectory.java:77)
   [junit4]    > 	at org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:99)
   [junit4]    > 	at org.apache.lucene.codecs.blocktree.BlockTreeTermsReader.<init>(BlockTreeTermsReader.java:153)
   [junit4]    > 	at org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.fieldsProducer(Lucene50PostingsFormat.java:445)
   [junit4]    > 	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:261)
   [junit4]    > 	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:341)
   [junit4]    > 	at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:106)
   [junit4]    > 	at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:74)
   [junit4]    > 	at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:145)
   [junit4]    > 	at org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:197)
   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:103)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:460)
   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader.doOpenFromWriter(StandardDirectoryReader.java:291)
   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:266)
   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader.doOpenIfChanged(StandardDirectoryReader.java:256)
   [junit4]    > 	at org.apache.lucene.index.DirectoryReader.openIfChanged(DirectoryReader.java:140)
   [junit4]    > 	at org.apache.lucene.index.TestIndexWriterReader.testDuringAddDelete(TestIndexWriterReader.java:870)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J2/temp/lucene.index.TestIndexWriterReader_61A624ED3323A1DA-001
   [junit4]   2> Sep 07, 2016 4:42:35 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 5 leaked thread(s).
   [junit4]   2> Sep 07, 2016 4:42:55 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 5 threads leaked from SUITE scope at org.apache.lucene.index.TestIndexWriterReader: 
   [junit4]   2>    1) Thread[id=1799, name=Thread-1549, state=BLOCKED, group=TGRP-TestIndexWriterReader]
   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   [junit4]   2>    2) Thread[id=1801, name=Thread-1551, state=RUNNABLE, group=TGRP-TestIndexWriterReader]
   [junit4]   2>         at org.apache.lucene.codecs.compressing.CompressingTermVectorsReader.get(CompressingTermVectorsReader.java:417)
   [junit4]   2>         at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat$AssertingTermVectorsReader.get(AssertingTermVectorsFormat.java:71)
   [junit4]   2>         at org.apache.lucene.codecs.TermVectorsWriter.merge(TermVectorsWriter.java:223)
   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.mergeVectors(SegmentMerger.java:210)
   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:144)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4287)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3864)
   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   [junit4]   2>    3) Thread[id=1802, name=Thread-1552, state=BLOCKED, group=TGRP-TestIndexWriterReader]
   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   [junit4]   2>    4) Thread[id=1800, name=Thread-1550, state=BLOCKED, group=TGRP-TestIndexWriterReader]
   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   [junit4]   2>    5) Thread[id=1798, name=Thread-1548, state=BLOCKED, group=TGRP-TestIndexWriterReader]
   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
   [junit4]   2>         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   [junit4]   2> Sep 07, 2016 4:42:55 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=1799, name=Thread-1549, state=BLOCKED, group=TGRP-TestIndexWriterReader]
   [junit4]   2>    2) Thread[id=1801, name=Thread-1551, state=RUNNABLE, group=TGRP-TestIndexWriterReader]
   [junit4]   2>    3) Thread[id=1802, name=Thread-1552, state=BLOCKED, group=TGRP-TestIndexWriterReader]
   [junit4]   2>    4) Thread[id=1800, name=Thread-1550, state=BLOCKED, group=TGRP-TestIndexWriterReader]
   [junit4]   2>    5) Thread[id=1798, name=Thread-1548, state=BLOCKED, group=TGRP-TestIndexWriterReader]
   [junit4]   2> Sep 07, 2016 4:42:55 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: All leaked threads terminated.
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {field1=Lucene50(blocksize=128), field=Lucene50(blocksize=128), indexname=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), foo=Lucene50(blocksize=128), id=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), field3=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), field2=FSTOrd50, field5=Lucene50(blocksize=128), field4=Lucene50(blocksize=128)}, docValues:{}, maxPointsInLeafNode=1314, maxMBSortInHeap=5.621346163809926, sim=RandomSimilarity(queryNorm=true): {field1=DFR I(F)B1, field=DFR I(n)L2, indexname=DFR GLZ(0.3), foo=DFR GLZ(0.3), field3=DFR I(ne)Z(0.3), field2=IB SPL-D1, field5=DFR I(F)1, field4=DFR I(F)LZ(0.3)}, locale=mk, timezone=Europe/Skopje
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=139897912,total=268959744
   [junit4]   2> NOTE: All tests run in this JVM: [Test2BPagedBytes, TestStressIndexing, TestGeoUtils, TestNoMergePolicy, TestRollback, TestBlockPostingsFormat, TestTermQuery, TestTopDocsMerge, TestFilterIterator, TestStressDeletes, TestIntsRef, TestDocBoost, TestNamedSPILoader, TestTransactions, TestRollingBuffer, TestFilterWeight, TestIndexWriterThreadsToSegments, TestLogMergePolicy, TestHighCompressionMode, TestDocValuesIndexing, TestRegexpRandom2, TestSingleInstanceLockFactory, TestLucene62SegmentInfoFormat, TestSegmentMerger, TestIndexWriterCommit, TestNRTCachingDirectory, TestAllFilesHaveCodecHeader, TestFuzzyQuery, Test2BPostings, TestDocumentsWriterDeleteQueue, TestByteBlockPool, TestScorerPerf, TestPayloadsOnVectors, TestParallelLeafReader, TestSegmentTermDocs, TestParallelTermEnum, TestDocument, TestVersion, TestAtomicUpdate, TestConjunctionDISI, TestBagOfPostings, TestMultiTermConstantScore, TestSnapshotDeletionPolicy, TestMmapDirectory, TestSpansEnum, TestSpanNotQuery, TestTimSorter, TestWildcardRandom, TestInPlaceMergeSorter, TestFixedBitDocIdSet, TestSimilarity, TestSearchWithThreads, TestCharacterUtils, TestUpgradeIndexMergePolicy, TestBlendedTermQuery, TestBooleanQuery, TestArrayUtil, TestSimpleSearchEquivalence, TestSpanMultiTermQueryWrapper, TestDeletionPolicy, TestMultiPhraseEnum, TestLazyProxSkipping, TestForUtil, TestSpanExplanations, TestReaderWrapperDVTypeCheck, TestFastCompressionMode, TestRegexpQuery, TestTopFieldCollector, TestMSBRadixSorter, TestIndexWriterMerging, TestFileSwitchDirectory, TestIndexWriter, TestIndexWriterReader]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestIndexWriterReader -Dtests.seed=61A624ED3323A1DA -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=mk -Dtests.timezone=Europe/Skopje -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J2 | TestIndexWriterReader (suite) <<<
   [junit4]    > Throwable #1: com.carrotsearch.randomizedtesting.ThreadLeakError: 5 threads leaked from SUITE scope at org.apache.lucene.index.TestIndexWriterReader: 
   [junit4]    >    1) Thread[id=1799, name=Thread-1549, state=BLOCKED, group=TGRP-TestIndexWriterReader]
   [junit4]    >         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
   [junit4]    >         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
   [junit4]    >         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   [junit4]    >    2) Thread[id=1801, name=Thread-1551, state=RUNNABLE, group=TGRP-TestIndexWriterReader]
   [junit4]    >         at org.apache.lucene.codecs.compressing.CompressingTermVectorsReader.get(CompressingTermVectorsReader.java:417)
   [junit4]    >         at org.apache.lucene.codecs.asserting.AssertingTermVectorsFormat$AssertingTermVectorsReader.get(AssertingTermVectorsFormat.java:71)
   [junit4]    >         at org.apache.lucene.codecs.TermVectorsWriter.merge(TermVectorsWriter.java:223)
   [junit4]    >         at org.apache.lucene.index.SegmentMerger.mergeVectors(SegmentMerger.java:210)
   [junit4]    >         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:144)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:4287)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3864)
   [junit4]    >         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
   [junit4]    >         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
   [junit4]    >         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   [junit4]    >    3) Thread[id=1802, name=Thread-1552, state=BLOCKED, group=TGRP-TestIndexWriterReader]
   [junit4]    >         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
   [junit4]    >         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
   [junit4]    >         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   [junit4]    >    4) Thread[id=1800, name=Thread-1550, state=BLOCKED, group=TGRP-TestIndexWriterReader]
   [junit4]    >         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
   [junit4]    >         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
   [junit4]    >         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   [junit4]    >    5) Thread[id=1798, name=Thread-1548, state=BLOCKED, group=TGRP-TestIndexWriterReader]
   [junit4]    >         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:37)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:2052)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4910)
   [junit4]    >         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:731)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4948)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4939)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1565)
   [junit4]    >         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
   [junit4]    >         at org.apache.lucene.index.TestIndexWriterReader$2.run(TestIndexWriterReader.java:848)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([61A624ED3323A1DA]:0)
   [junit4] Completed [200/432 (1!)] on J2 in 252.56s, 23 tests, 2 errors <<< FAILURES!

[...truncated 10658 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.StressHdfsTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.hdfs.StressHdfsTest_66DE89DEA79A3E32-001/init-core-data-001
   [junit4]   2> 814899 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=https://issues.apache.org/jira/browse/SOLR-5776)
   [junit4]   2> 814899 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /_a/m
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 815026 WARN  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 815069 WARN  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 815081 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 815125 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_33702_hdfs____13ay83/webapp
   [junit4]   2> 816226 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:33702
   [junit4]   2> 817177 WARN  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 817179 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 817200 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_46024_datanode____.2mcfop/webapp
   [junit4]   2> 818238 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:46024
   [junit4]   2> 818469 WARN  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 818470 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 818502 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_55265_datanode____.v9bv0o/webapp
   [junit4]   2> 818789 INFO  (IPC Server handler 6 on 59517) [    ] BlockStateChange BLOCK* processReport: from storage DS-124aedd7-53e1-4abe-9593-1749020619f8 node DatanodeRegistration(127.0.0.1:39806, datanodeUuid=44048ab0-2317-4266-b18a-947e4bc5cd29, infoPort=51796, infoSecurePort=0, ipcPort=39160, storageInfo=lv=-56;cid=testClusterID;nsid=2101821981;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 818789 INFO  (IPC Server handler 6 on 59517) [    ] BlockStateChange BLOCK* processReport: from storage DS-0427d4bd-1eeb-4af5-a408-0b312310f3bb node DatanodeRegistration(127.0.0.1:39806, datanodeUuid=44048ab0-2317-4266-b18a-947e4bc5cd29, infoPort=51796, infoSecurePort=0, ipcPort=39160, storageInfo=lv=-56;cid=testClusterID;nsid=2101821981;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 819366 INFO  (SUITE-StressHdfsTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:55265
   [junit4]   2> 819712 INFO  (IPC Server handler 3 on 59517) [    ] BlockStateChange BLOCK* processReport: from storage DS-0a413b87-a6df-47f1-b731-f3928bdb7bd0 node DatanodeRegistration(127.0.0.1:38537, datanodeUuid=6109fe6a-418b-4810-8965-110624cd978c, infoPort=48162, infoSecurePort=0, ipcPort=33212, storageInfo=lv=-56;cid=testClusterID;nsid=2101821981;c=0), blocks: 0, hasStaleStorage: true, processing time: 86 msecs
   [junit4]   2> 819712 INFO  (IPC Server handler 3 on 59517) [    ] BlockStateChange BLOCK* processReport: from storage DS-394f120c-9225-45db-9d4d-a79e0357d0a0 node DatanodeRegistration(127.0.0.1:38537, datanodeUuid=6109fe6a-418b-4810-8965-110624cd978c, infoPort=48162, infoSecurePort=0, ipcPort=33212, storageInfo=lv=-56;cid=testClusterID;nsid=2101821981;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 819976 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 819977 INFO  (Thread-1303) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 819977 INFO  (Thread-1303) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 820180 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.ZkTestServer start zk server on port:43044
   [junit4]   2> 820180 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 820189 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 820201 INFO  (zkCallback-698-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6ea7009c name:ZooKeeperConnection Watcher:127.0.0.1:43044 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 820201 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 820201 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 820201 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 820210 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 820215 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 820239 INFO  (zkCallback-699-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@1095833c name:ZooKeeperConnection Watcher:127.0.0.1:43044/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 820241 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 820241 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 820241 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1
   [junit4]   2> 820242 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/shards
   [junit4]   2> 820244 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection
   [junit4]   2> 820244 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection/shards
   [junit4]   2> 820250 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 820250 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 820252 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 820252 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/schema.xml
   [junit4]   2> 820253 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 820253 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 820255 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 820255 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 820262 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 820262 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/protwords.txt
   [junit4]   2> 820263 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 820263 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/currency.xml
   [junit4]   2> 820264 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 820264 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 820270 INFO  (TEST-StressHdfsTest.test-seed#[66DE89DEA79A3E32]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenki

[...truncated too long message...]

ctory Closing directory, CoreContainer#isShutdown=true
   [junit4]   2> 4696298 INFO  (coreCloseExecutor-5736-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory Closing directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedFacetPivotLongTailTest_66DE89DEA79A3E32-001/tempDir-001/shard2/cores/collection1/data/index
   [junit4]   2> 4696305 INFO  (TEST-DistributedFacetPivotLongTailTest.test-seed#[66DE89DEA79A3E32]) [    ] o.e.j.s.h.ContextHandler Stopped o.e.j.s.ServletContextHandler@17e66fe8{/_/d,null,UNAVAILABLE}
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=DistributedFacetPivotLongTailTest -Dtests.method=test -Dtests.seed=66DE89DEA79A3E32 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=cs -Dtests.timezone=Etc/GMT-6 -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR    131s J0 | DistributedFacetPivotLongTailTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: https://127.0.0.1:46729/_/d/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([66DE89DEA79A3E32:EE8AB604096653CA]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:622)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:173)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:138)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:152)
   [junit4]    > 	at org.apache.solr.handler.component.DistributedFacetPivotLongTailTest.test(DistributedFacetPivotLongTailTest.java:71)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.conn.ConnectTimeoutException: Connect to 127.0.0.1:46729 [/127.0.0.1] failed: Read timed out
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
   [junit4]    > 	at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
   [junit4]    > 	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
   [junit4]    > 	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
   [junit4]    > 	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
   [junit4]    > 	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
   [junit4]    > 	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: java.net.SocketTimeoutException: Read timed out
   [junit4]    > 	at java.net.SocketInputStream.socketRead0(Native Method)
   [junit4]    > 	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:170)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:141)
   [junit4]    > 	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
   [junit4]    > 	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
   [junit4]    > 	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
   [junit4]    > 	at sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1375)
   [junit4]    > 	at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1403)
   [junit4]    > 	at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1387)
   [junit4]    > 	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.createLayeredSocket(SSLConnectionSocketFactory.java:394)
   [junit4]    > 	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:353)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:134)
   [junit4]    > 	... 58 more
   [junit4]   2> 4696316 INFO  (SUITE-DistributedFacetPivotLongTailTest-seed#[66DE89DEA79A3E32]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.component.DistributedFacetPivotLongTailTest_66DE89DEA79A3E32-001
   [junit4]   2> NOTE: test params are: codec=Lucene62, sim=ClassicSimilarity, locale=cs, timezone=Etc/GMT-6
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=251613816,total=477626368
   [junit4]   2> NOTE: All tests run in this JVM: [OverseerModifyCollectionTest, RequiredFieldsTest, DistributedDebugComponentTest, TestMissingGroups, TestIBSimilarityFactory, CdcrBootstrapTest, TestManagedSynonymFilterFactory, LeaderInitiatedRecoveryOnShardRestartTest, TestSubQueryTransformerCrossCore, TestOnReconnectListenerSupport, ZkStateReaderTest, TestSolrQueryParserResource, ResourceLoaderTest, TestConfigSetImmutable, TestSolr4Spatial, TestFreeTextSuggestions, TestReplicationHandlerBackup, ReturnFieldsTest, TestClassicSimilarityFactory, HdfsThreadLeakTest, SharedFSAutoReplicaFailoverUtilsTest, TestCSVLoader, LeaderElectionIntegrationTest, ZkNodePropsTest, TestRemoteStreaming, TestSchemaManager, EnumFieldTest, BasicZkTest, DirectoryFactoryTest, RemoteQueryErrorTest, SOLR749Test, TestLMDirichletSimilarityFactory, TestStressUserVersions, TestFieldCacheSortRandom, TestDistribDocBasedVersion, ChangedSchemaMergeTest, RuleEngineTest, TestPhraseSuggestions, TestCloudPseudoReturnFields, TestReplicationHandler, DisMaxRequestHandlerTest, TestLegacyFieldCache, TestComplexPhraseQParserPlugin, BasicDistributedZkTest, TestLRUCache, TestDistributedSearch, RestartWhileUpdatingTest, TestMergePolicyConfig, TestRebalanceLeaders, TestInitParams, CacheHeaderTest, BitVectorTest, TestArbitraryIndexDir, TestCloudDeleteByQuery, PathHierarchyTokenizerFactoryTest, TestSchemaNameResource, TestOrdValues, DistributedFacetPivotLargeTest, BlockJoinFacetDistribTest, TestFaceting, SynonymTokenizerTest, SpatialHeatmapFacetsTest, TestMiniSolrCloudClusterBase, OutOfBoxZkACLAndCredentialsProvidersTest, QueryEqualityTest, IndexSchemaTest, TestBadConfig, TestAddFieldRealTimeGet, BlockDirectoryTest, DefaultValueUpdateProcessorTest, TestCollationFieldDocValues, TestFoldingMultitermQuery, TestBlobHandler, SolrCloudExampleTest, DistributedSpellCheckComponentTest, ConnectionReuseTest, PingRequestHandlerTest, DocumentAnalysisRequestHandlerTest, AddSchemaFieldsUpdateProcessorFactoryTest, HdfsCollectionsAPIDistributedZkTest, TestRTGBase, CloneFieldUpdateProcessorFactoryTest, TestSizeLimitedDistributedMap, TestLuceneMatchVersion, TestRawResponseWriter, SignatureUpdateProcessorFactoryTest, TestXmlQParserPlugin, TestComponentsName, TestSQLHandler, HdfsLockFactoryTest, TestIntervalFaceting, TestCloudManagedSchema, TestDynamicFieldCollectionResource, TestSortingResponseWriter, TestHighlightDedupGrouping, UUIDFieldTest, ConnectionManagerTest, AnalysisAfterCoreReloadTest, TestSchemaVersionResource, JSONWriterTest, TestUninvertingReader, TermVectorComponentDistributedTest, TestGroupingSearch, SimpleCollectionCreateDeleteTest, DistributedFacetPivotLongTailTest]
   [junit4] Completed [632/633 (5!)] on J0 in 130.67s, 1 test, 1 error <<< FAILURES!

[...truncated 1 lines...]
   [junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20160907_083928_132.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid30315.hprof ...
   [junit4] Heap dump file created [707246725 bytes in 19.813 secs]
   [junit4] <<< JVM J0: EOF ----

[...truncated 11039 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:715: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid30315.hprof

Total time: 367 minutes 35 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1110 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1110/

11 tests failed.
FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testUpdateLogSynchronisation

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard2

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard2
	at __randomizedtesting.SeedInfo.seed([C7E4E06A8290DA3E:398BB8C940B0F92F]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testUpdateLogSynchronisation(CdcrReplicationDistributedZkTest.java:377)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchAddsWithDelete

Error Message:
Timeout while trying to assert number of documents @ target_collection

Stack Trace:
java.lang.AssertionError: Timeout while trying to assert number of documents @ target_collection
	at __randomizedtesting.SeedInfo.seed([C7E4E06A8290DA3E:B0FE99CE2A89DB12]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:273)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchAddsWithDelete(CdcrReplicationDistributedZkTest.java:532)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.AssertionError: expected:<59> but was:<0>
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.assertNumDocs(BaseCdcrDistributedZkTest.java:264)
	... 42 more


FAILED:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=43351, name=Thread-27331, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=43351, name=Thread-27331, state=RUNNABLE, group=TGRP-FullSolrCloudDistribCmdsTest]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:44445/yx/eg/collection1
	at __randomizedtesting.SeedInfo.seed([C7E4E06A8290DA3E]:0)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:644)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:44445/yx/eg/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:166)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest$1IndexThread.run(FullSolrCloudDistribCmdsTest.java:642)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 5 more


FAILED:  org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.test

Error Message:
Error from server at https://127.0.0.1:33378/h_ej/rc: KeeperErrorCode = NoNode for /overseer/collection-queue-work/qnr-0000000150

Stack Trace:
org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at https://127.0.0.1:33378/h_ej/rc: KeeperErrorCode = NoNode for /overseer/collection-queue-work/qnr-0000000150
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:608)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1599)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.createCollection(AbstractFullDistribZkTestBase.java:1620)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.deleteCollectionWithDownNodes(CollectionsAPIDistributedZkTest.java:345)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:186)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.core.OpenCloseCoreStressTest.test10Minutes

Error Message:
Captured an uncaught exception in thread: Thread[id=22510, name=Thread-13374, state=RUNNABLE, group=TGRP-OpenCloseCoreStressTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=22510, name=Thread-13374, state=RUNNABLE, group=TGRP-OpenCloseCoreStressTest]
Caused by: java.lang.OutOfMemoryError: GC overhead limit exceeded
	at __randomizedtesting.SeedInfo.seed([C7E4E06A8290DA3E]:0)
	at sun.security.ssl.InputRecord.<init>(InputRecord.java:93)
	at sun.security.ssl.AppInputStream.<init>(AppInputStream.java:50)
	at sun.security.ssl.SSLSocketImpl.init(SSLSocketImpl.java:635)
	at sun.security.ssl.SSLSocketImpl.<init>(SSLSocketImpl.java:567)
	at sun.security.ssl.SSLSocketFactoryImpl.createSocket(SSLSocketFactoryImpl.java:110)
	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.createLayeredSocket(SSLConnectionSocketFactory.java:363)
	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:353)
	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:134)
	at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
	at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:942)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:957)
	at org.apache.solr.core.OneQuery.run(OpenCloseCoreStressTest.java:502)


FAILED:  org.apache.solr.handler.component.DistributedFacetPivotLongTailTest.test

Error Message:
GC overhead limit exceeded

Stack Trace:
java.lang.OutOfMemoryError: GC overhead limit exceeded
	at __randomizedtesting.SeedInfo.seed([C7E4E06A8290DA3E:4FB0DFB02C6CB7C6]:0)
	at java.util.HashMap.resize(HashMap.java:703)
	at java.util.HashMap.putVal(HashMap.java:628)
	at java.util.HashMap.put(HashMap.java:611)
	at java.util.HashSet.add(HashSet.java:219)
	at java.util.AbstractCollection.addAll(AbstractCollection.java:344)
	at sun.security.ssl.SSLAlgorithmDecomposer.decompose(SSLAlgorithmDecomposer.java:212)
	at sun.security.ssl.SSLAlgorithmDecomposer.decompose(SSLAlgorithmDecomposer.java:243)
	at sun.security.util.AbstractAlgorithmConstraints.checkAlgorithm(AbstractAlgorithmConstraints.java:104)
	at sun.security.util.DisabledAlgorithmConstraints.permits(DisabledAlgorithmConstraints.java:91)
	at sun.security.ssl.SSLAlgorithmConstraints.permits(SSLAlgorithmConstraints.java:149)
	at sun.security.ssl.Handshaker.getActiveCipherSuites(Handshaker.java:642)
	at sun.security.ssl.Handshaker.activate(Handshaker.java:509)
	at sun.security.ssl.SSLSocketImpl.kickstartHandshake(SSLSocketImpl.java:1482)
	at sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1351)
	at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1403)
	at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1387)
	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.createLayeredSocket(SSLConnectionSocketFactory.java:394)
	at org.apache.http.conn.ssl.SSLConnectionSocketFactory.connectSocket(SSLConnectionSocketFactory.java:353)
	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:134)
	at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
	at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:149)


FAILED:  org.apache.solr.security.BasicAuthIntegrationTest.testBasics

Error Message:
IOException occured when talking to server at: http://127.0.0.1:57585/solr/testSolrCloudCollection_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: IOException occured when talking to server at: http://127.0.0.1:57585/solr/testSolrCloudCollection_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([C7E4E06A8290DA3E:FA3C4E46BA7E844E]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:755)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1161)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1050)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:992)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.security.BasicAuthIntegrationTest.doExtraTests(BasicAuthIntegrationTest.java:193)
	at org.apache.solr.cloud.TestMiniSolrCloudClusterBase.testCollectionCreateSearchDelete(TestMiniSolrCloudClusterBase.java:196)
	at org.apache.solr.cloud.TestMiniSolrCloudClusterBase.testBasics(TestMiniSolrCloudClusterBase.java:79)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57585/solr/testSolrCloudCollection_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:622)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:415)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:367)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:730)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:57585 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 10 more


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchBoundaries

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard1

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard1
	at __randomizedtesting.SeedInfo.seed([C7E4E06A8290DA3E:E5CB393FFB341BD9]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchBoundaries(CdcrReplicationDistributedZkTest.java:557)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:50706/k_qp/b

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:50706/k_qp/b
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:477)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
	at __randomizedtesting.SeedInfo.seed([C7E4E06A8290DA3E]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest:     1) Thread[id=31388, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=31358, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=31368, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=31373, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=31393, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=31353, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=31348, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=31378, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=31363, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=31383, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 
   1) Thread[id=31388, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=31358, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=31368, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=31373, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=31393, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=31353, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=31348, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=31378, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=31363, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=31383, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([C7E4E06A8290DA3E]:0)




Build Log:
[...truncated 11527 lines...]
   [junit4] Suite: org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C7E4E06A8290DA3E-001/init-core-data-001
   [junit4]   2> 350938 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 350939 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /h_ej/rc
   [junit4]   1> Formatting using clusterid: testClusterID
   [junit4]   2> 351035 WARN  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.a.h.m.i.MetricsConfig Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties
   [junit4]   2> 351051 WARN  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 351058 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 351088 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/hdfs to ./temp/Jetty_localhost_49283_hdfs____upt5te/webapp
   [junit4]   2> 351660 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:49283
   [junit4]   2> 351754 WARN  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 351755 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 351769 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_37236_datanode____.1na3xk/webapp
   [junit4]   2> 352484 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:37236
   [junit4]   2> 352831 WARN  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.a.h.h.HttpRequestLog Jetty request log can only be enabled using Log4j
   [junit4]   2> 352832 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.m.log jetty-6.1.26
   [junit4]   2> 352861 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.m.log Extract jar:file:/x1/jenkins/.ivy2/cache/org.apache.hadoop/hadoop-hdfs/tests/hadoop-hdfs-2.7.2-tests.jar!/webapps/datanode to ./temp/Jetty_localhost_48061_datanode____wmx0em/webapp
   [junit4]   2> 353096 INFO  (IPC Server handler 2 on 43110) [    ] BlockStateChange BLOCK* processReport: from storage DS-e2653e2a-8c7e-4133-818f-4bb20db57ef8 node DatanodeRegistration(127.0.0.1:40487, datanodeUuid=e4c267d3-903c-47ed-a021-0e4e34b9c543, infoPort=34091, infoSecurePort=0, ipcPort=53589, storageInfo=lv=-56;cid=testClusterID;nsid=1359521124;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 353096 INFO  (IPC Server handler 2 on 43110) [    ] BlockStateChange BLOCK* processReport: from storage DS-05280b5b-55ed-4af8-8cf4-55b99d1d8d26 node DatanodeRegistration(127.0.0.1:40487, datanodeUuid=e4c267d3-903c-47ed-a021-0e4e34b9c543, infoPort=34091, infoSecurePort=0, ipcPort=53589, storageInfo=lv=-56;cid=testClusterID;nsid=1359521124;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 353462 INFO  (SUITE-HdfsCollectionsAPIDistributedZkTest-seed#[C7E4E06A8290DA3E]-worker) [    ] o.m.log Started HttpServer2$SelectChannelConnectorWithSafeStartup@localhost:48061
   [junit4]   2> 353939 INFO  (IPC Server handler 4 on 43110) [    ] BlockStateChange BLOCK* processReport: from storage DS-5330f517-3544-4622-958e-bf2817ec78ae node DatanodeRegistration(127.0.0.1:60255, datanodeUuid=30b4148e-f09b-439d-b9f7-20fa3220b2b0, infoPort=54558, infoSecurePort=0, ipcPort=51718, storageInfo=lv=-56;cid=testClusterID;nsid=1359521124;c=0), blocks: 0, hasStaleStorage: true, processing time: 0 msecs
   [junit4]   2> 353939 INFO  (IPC Server handler 4 on 43110) [    ] BlockStateChange BLOCK* processReport: from storage DS-44dc2c54-df24-4aa9-bee0-8edabac13b64 node DatanodeRegistration(127.0.0.1:60255, datanodeUuid=30b4148e-f09b-439d-b9f7-20fa3220b2b0, infoPort=54558, infoSecurePort=0, ipcPort=51718, storageInfo=lv=-56;cid=testClusterID;nsid=1359521124;c=0), blocks: 0, hasStaleStorage: false, processing time: 0 msecs
   [junit4]   2> 354051 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 354054 INFO  (Thread-707) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 354054 INFO  (Thread-707) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 354154 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.ZkTestServer start zk server on port:39994
   [junit4]   2> 354154 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 354182 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 354190 INFO  (zkCallback-408-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@790a925b name:ZooKeeperConnection Watcher:127.0.0.1:39994 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 354191 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 354191 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 354191 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 354194 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 354199 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 354204 INFO  (zkCallback-409-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5662b9c0 name:ZooKeeperConnection Watcher:127.0.0.1:39994/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 354204 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 354205 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 354205 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1
   [junit4]   2> 354206 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/shards
   [junit4]   2> 354208 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection
   [junit4]   2> 354209 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection/shards
   [junit4]   2> 354210 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 354210 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 354213 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 354213 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/schema.xml
   [junit4]   2> 354214 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 354215 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 354216 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 354216 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 354218 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 354218 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/protwords.txt
   [junit4]   2> 354219 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 354219 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/currency.xml
   [junit4]   2> 354221 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 354221 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 354222 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 354222 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 354224 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 354224 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 354225 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 354225 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 354227 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 354227 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 354230 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 354230 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 354242 INFO  (zkCallback-411-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@592a7ee9 name:ZooKeeperConnection Watcher:127.0.0.1:39994/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 354242 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 354242 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 354244 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 354245 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 354252 INFO  (zkCallback-412-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2249b59d name:ZooKeeperConnection Watcher:127.0.0.1:39994 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 354253 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 354253 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 354253 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 354255 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 354255 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 354261 INFO  (zkCallback-413-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@4925e88b name:ZooKeeperConnection Watcher:127.0.0.1:39994/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 354261 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 354261 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 354261 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf2/solrconfig.xml
   [junit4]   2> 354261 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/solrconfig.xml
   [junit4]   2> 354264 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf2/schema.xml
   [junit4]   2> 354264 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/schema.xml
   [junit4]   2> 354266 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf2/enumsConfig.xml
   [junit4]   2> 354266 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/enumsConfig.xml
   [junit4]   2> 354267 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf2/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 354267 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 354269 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf2/stopwords.txt
   [junit4]   2> 354269 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/stopwords.txt
   [junit4]   2> 354270 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf2/protwords.txt
   [junit4]   2> 354271 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/protwords.txt
   [junit4]   2> 354272 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf2/currency.xml
   [junit4]   2> 354272 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/currency.xml
   [junit4]   2> 354274 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf2/open-exchange-rates.json
   [junit4]   2> 354274 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/open-exchange-rates.json
   [junit4]   2> 354275 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf2/mapping-ISOLatin1Accent.txt
   [junit4]   2> 354275 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/mapping-ISOLatin1Accent.txt
   [junit4]   2> 354277 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf2/old_synonyms.txt
   [junit4]   2> 354277 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/old_synonyms.txt
   [junit4]   2> 354278 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf2/synonyms.txt
   [junit4]   2> 354278 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/synonyms.txt
   [junit4]   2> 354280 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/elevate.xml to /configs/conf2/elevate.xml
   [junit4]   2> 354280 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf2/elevate.xml
   [junit4]   2> 354384 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C7E4E06A8290DA3E-001/control-001/cores/collection1
   [junit4]   2> 354387 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 354388 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4840ebea{/h_ej/rc,null,AVAILABLE}
   [junit4]   2> 354390 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.e.j.s.ServerConnector Started ServerConnector@545dcd98{SSL,[ssl, http/1.1]}{127.0.0.1:59258}
   [junit4]   2> 354390 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.e.j.s.Server Started @358914ms
   [junit4]   2> 354390 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=hdfs://localhost:43110/hdfs__localhost_43110__x1_jenkins_jenkins-slave_workspace_Lucene-Solr-NightlyTests-master_checkout_solr_build_solr-core_test_J0_temp_solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C7E4E06A8290DA3E-001_tempDir-002_control_data, hostContext=/h_ej/rc, hostPort=59258, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C7E4E06A8290DA3E-001/control-001/cores}
   [junit4]   2> 354391 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 354391 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C7E4E06A8290DA3E-001/control-001'
   [junit4]   2> 354391 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 354391 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 354391 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 354398 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 354422 INFO  (zkCallback-414-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2ca4a4fa name:ZooKeeperConnection Watcher:127.0.0.1:39994/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 354422 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 354422 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 354427 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 354427 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C7E4E06A8290DA3E-001/control-001/solr.xml
   [junit4]   2> 354432 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C7E4E06A8290DA3E-001/control-001/cores
   [junit4]   2> 354433 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.CoreContainer New CoreContainer 943977700
   [junit4]   2> 354433 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C7E4E06A8290DA3E-001/control-001]
   [junit4]   2> 354433 WARN  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.CoreContainer Couldn't add files from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C7E4E06A8290DA3E-001/control-001/lib to classpath: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest_C7E4E06A8290DA3E-001/control-001/lib
   [junit4]   2> 354433 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,
   [junit4]   2> 354435 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.u.UpdateShardHandler Created UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&maxConnections=100000&maxConnectionsPerHost=100000
   [junit4]   2> 354435 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 354435 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 354435 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:39994/solr
   [junit4]   2> 354436 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 354436 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 354438 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 354451 INFO  (zkCallback-417-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@2890341e name:ZooKeeperConnection Watcher:127.0.0.1:39994 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 354454 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 354454 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 354455 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$65/1942881181@4c08a61c
   [junit4]   2> 354456 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 354465 INFO  (zkCallback-418-thread-1-processing-n:127.0.0.1:59258_h_ej%2Frc) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@5609ff6a name:ZooKeeperConnection Watcher:127.0.0.1:39994/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 354465 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 354466 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
   [junit4]   2> 354468 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
   [junit4]   2> 354471 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
   [junit4]   2> 354473 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
   [junit4]   2> 354475 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
   [junit4]   2> 354476 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 354478 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 354484 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 354485 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 354486 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 354487 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.ZkStateReader Loaded cluster properties: {urlScheme=https}
   [junit4]   2> 354487 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 354489 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
   [junit4]   2> 354491 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
   [junit4]   2> 354492 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 354492 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/96524164962320391-127.0.0.1:59258_h_ej%2Frc-n_0000000000
   [junit4]   2> 354492 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:59258_h_ej%2Frc
   [junit4]   2> 354492 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
   [junit4]   2> 354493 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.Overseer Overseer (id=96524164962320391-127.0.0.1:59258_h_ej%2Frc-n_0000000000) starting
   [junit4]   2> 354495 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
   [junit4]   2> 354501 INFO  (TEST-HdfsCollectionsAPIDistributedZkTest.test-seed#[C7E4E06A8290DA3E]) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=10000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 354501 INFO  (OverseerCollectionConfigSetProcessor-96524164962320391-127.0.0.1:59258_h_ej%2Frc-n_0000000000) [n:127.0.0.1:59258_h_ej%2Frc    ] o.a.s.c.O

[...truncated too long message...]

.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
   [junit4]    > 	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
   [junit4]    > 	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
   [junit4]    > 	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
   [junit4]    > 	... 10 more
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.security.BasicAuthIntegrationTest_C7E4E06A8290DA3E-001
   [junit4]   2> Sep 02, 2016 8:26:35 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {range_facet_l_dv=PostingsFormat(name=Memory doPackFST= true), _version_=FST50, multiDefault=PostingsFormat(name=LuceneFixedGap), intDefault=FST50, id=PostingsFormat(name=Memory doPackFST= true), range_facet_i_dv=FST50, range_facet_l=FST50, timestamp=FST50}, docValues:{range_facet_l_dv=DocValuesFormat(name=Asserting), range_facet_i_dv=DocValuesFormat(name=Memory), timestamp=DocValuesFormat(name=Memory)}, maxPointsInLeafNode=801, maxMBSortInHeap=6.596519607923508, sim=RandomSimilarity(queryNorm=true): {}, locale=fr-CH, timezone=Africa/Windhoek
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=172398656,total=520093696
   [junit4]   2> NOTE: All tests run in this JVM: [TestBackupRepositoryFactory, AliasIntegrationTest, NumericFieldsTest, TestManagedSchemaAPI, SecurityConfHandlerTest, SpatialFilterTest, TestSolrCoreSnapshots, TestCollectionAPI, ExitableDirectoryReaderTest, SuggestComponentTest, ShardRoutingTest, TestTrie, DocValuesMissingTest, ZkStateWriterTest, SpellCheckComponentTest, HighlighterTest, AsyncMigrateRouteKeyTest, CoreAdminRequestStatusTest, WordBreakSolrSpellCheckerTest, UniqFieldsUpdateProcessorFactoryTest, CollectionTooManyReplicasTest, TestBM25SimilarityFactory, HdfsTlogReplayBufferedWhileIndexingTest, SchemaVersionSpecificBehaviorTest, SolrCmdDistributorTest, TestRandomFlRTGCloud, TestSchemaSimilarityResource, VersionInfoTest, TestDownShardTolerantSearch, TestDefaultSearchFieldResource, MultiTermTest, SortByFunctionTest, ZkCLITest, TestSolrCLIRunExample, SimpleMLTQParserTest, OpenExchangeRatesOrgProviderTest, TestSolrQueryParserDefaultOperatorResource, ConvertedLegacyTest, TestFieldCollectionResource, AnalysisErrorHandlingTest, TestFieldSortValues, LoggingHandlerTest, ClassificationUpdateProcessorFactoryTest, DOMUtilTest, TestLeaderElectionZkExpiry, DistributedTermsComponentTest, BJQParserTest, DistributedQueryComponentCustomSortTest, CreateCollectionCleanupTest, TestDFRSimilarityFactory, TestReqParamsAPI, BasicFunctionalityTest, DistanceUnitsTest, TestExactStatsCache, TestPivotHelperCode, SimplePostToolTest, DeleteLastCustomShardedReplicaTest, SolrPluginUtilsTest, OverriddenZkACLAndCredentialsProvidersTest, TestFieldTypeResource, FacetPivotSmallTest, TestQuerySenderListener, PeerSyncReplicationTest, TestDefaultStatsCache, TestScoreJoinQPScore, TestCollapseQParserPlugin, TestMacros, TestDistributedGrouping, TestConfigSetsAPIZkFailure, NotRequiredUniqueKeyTest, BooleanFieldTest, VMParamsZkACLAndCredentialsProvidersTest, SpellCheckCollatorTest, FileUtilsTest, TermsComponentTest, TestFieldCacheWithThreads, TestNRTOpen, TestFieldCache, DistribDocExpirationUpdateProcessorTest, HdfsNNFailoverTest, TestConfigReload, TestSolrConfigHandlerCloud, ShardSplitTest, MinimalSchemaTest, UpdateRequestProcessorFactoryTest, TestSolrConfigHandler, SuggesterFSTTest, TestWordDelimiterFilterFactory, ZkSolrClientTest, TestNonDefinedSimilarityFactory, TestFastLRUCache, TestHashPartitioner, DistributedMLTComponentTest, TestSubQueryTransformer, SolrInfoMBeanTest, TestConfigSetProperties, TestSystemIdResolver, ChaosMonkeySafeLeaderTest, HdfsRestartWhileUpdatingTest, DocumentBuilderTest, SyncSliceTest, OpenCloseCoreStressTest, ClusterStateUpdateTest, TestZkChroot, TestRandomDVFaceting, ShardRoutingCustomTest, AutoCommitTest, TestFaceting, TestRecovery, DistributedSpellCheckComponentTest, TermVectorComponentDistributedTest, ZkControllerTest, TestRealTimeGet, TestJoin, TestReload, TestStressVersions, TestCoreContainer, SimpleFacetsTest, StatsComponentTest, PeerSyncTest, TestSort, TestFiltering, TestFunctionQuery, TestLazyCores, DirectUpdateHandlerTest, SoftAutoCommitTest, ShowFileRequestHandlerTest, CurrencyFieldOpenExchangeTest, SolrIndexSplitterTest, TestCoreDiscovery, SolrRequestParserTest, TestFoldingMultitermQuery, DocValuesTest, SuggesterTSTTest, TestStressLucene, SuggesterWFSTTest, PolyFieldTest, NoCacheHeaderTest, TestUpdate, TestAtomicUpdateErrorCases, QueryEqualityTest, StatelessScriptUpdateProcessorFactoryTest, DistanceFunctionTest, TestSolrDeletionPolicy1, XsltUpdateRequestHandlerTest, DebugComponentTest, CacheHeaderTest, IndexBasedSpellCheckerTest, TestSurroundQueryParser, DisMaxRequestHandlerTest, TestQueryUtils, DirectSolrSpellCheckerTest, DocumentAnalysisRequestHandlerTest, TestQueryTypes, TermVectorComponentTest, TestIndexingPerformance, MoreLikeThisHandlerTest, TestArbitraryIndexDir, FastVectorHighlighterTest, RegexBoostProcessorTest, TestJmxIntegration, ReturnFieldsTest, JSONWriterTest, TestCSVResponseWriter, MBeansHandlerTest, QueryParsingTest, TestAnalyzedSuggestions, JsonLoaderTest, BinaryUpdateRequestHandlerTest, TestPartialUpdateDeduplication, TestPHPSerializedResponseWriter, CSVRequestHandlerTest, TestComponentsName, SearchHandlerTest, TestBinaryResponseWriter, TestLFUCache, PreAnalyzedFieldTest, TestSuggestSpellingConverter, SpellingQueryConverterTest, RAMDirectoryFactoryTest, TestLRUCache, TestUtils, TestDocumentBuilder, ZkNodePropsTest, SliceStateTest, UUIDFieldTest, TestRTGBase, DistributedIntervalFacetingTest, CdcrVersionReplicationTest, CleanupOldIndexTest, CloudExitableDirectoryReaderTest, CollectionReloadTest, ConcurrentDeleteAndCreateCollectionTest, DistribCursorPagingTest, DistribJoinFromCollectionTest, HttpPartitionTest, MigrateRouteKeyTest, OverseerModifyCollectionTest, OverseerRolesTest, ReplaceNodeTest, RollingRestartTest, TestStressLiveNodes, TestTolerantUpdateProcessorCloud, TestTolerantUpdateProcessorRandomCloud, TriLevelCompositeIdRoutingTest, HdfsChaosMonkeyNothingIsSafeTest, HdfsUnloadDistributedZkTest, StressHdfsTest, CoreSorterTest, HdfsDirectoryFactoryTest, TestConfigSets, TestImplicitCoreProperties, TestInfoStreamLogging, TestShardHandlerFactory, RequestLoggingTest, TestReplicationHandlerBackup, CoreAdminCreateDiscoverTest, DistributedExpandComponentTest, DistributedQueryComponentOptimizationTest, JavabinLoaderTest, HighlighterMaxOffsetTest, TestSlowCompositeReaderWrapper, CheckHdfsIndexTest, TestCloudSchemaless, TestReloadDeadlock, TestSearcherReuse, TestSmileRequest, TestStandardQParsers, TestStressUserVersions, TestXmlQParser, TestMinMaxOnMultiValuedField, TestOrdValues, TestSortByMinMaxFunction, BlockJoinFacetDistribTest, CloudMLTQParserTest, BasicAuthIntegrationTest]
   [junit4] Completed [596/632 (8!)] on J1 in 18.87s, 1 test, 1 error <<< FAILURES!

[...truncated 113 lines...]
   [junit4] JVM J2: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J2-20160902_183833_957.sysout
   [junit4] >>> JVM J2 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: Java heap space
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid3526.hprof ...
   [junit4] Heap dump file created [491695500 bytes in 1.772 secs]
   [junit4] <<< JVM J2: EOF ----

[...truncated 3 lines...]
   [junit4] JVM J1: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J1-20160902_183833_951.sysout
   [junit4] >>> JVM J1 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid3521.hprof ...
   [junit4] Heap dump file created [560566947 bytes in 3.764 secs]
   [junit4] <<< JVM J1: EOF ----

[...truncated 9 lines...]
   [junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20160902_183833_950.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid3520.hprof ...
   [junit4] Heap dump file created [488680050 bytes in 3.901 secs]
   [junit4] <<< JVM J0: EOF ----

[...truncated 11036 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:715: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid3520.hprof
* java_pid3521.hprof
* java_pid3526.hprof

Total time: 318 minutes 23 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1109 - Failure

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1109/

10 tests failed.
FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testResilienceWithDeleteByQueryOnTarget

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard1

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard1
	at __randomizedtesting.SeedInfo.seed([CAAE2B1DF4390888:6ACCB9872460CDCD]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testResilienceWithDeleteByQueryOnTarget(CdcrReplicationDistributedZkTest.java:588)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchBoundaries

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard2

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard2
	at __randomizedtesting.SeedInfo.seed([CAAE2B1DF4390888:E881F2488D9DC96F]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testBatchBoundaries(CdcrReplicationDistributedZkTest.java:558)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable

Error Message:
Timeout while trying to assert replication errors

Stack Trace:
java.lang.AssertionError: Timeout while trying to assert replication errors
	at __randomizedtesting.SeedInfo.seed([CAAE2B1DF4390888:2C3A716EFF6665A0]:0)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable(CdcrReplicationDistributedZkTest.java:145)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.AssertionError
	at org.junit.Assert.fail(Assert.java:92)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertTrue(Assert.java:54)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testTargetCollectionNotAvailable(CdcrReplicationDistributedZkTest.java:132)
	... 41 more


FAILED:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Timeout occured while waiting response from server at: https://127.0.0.1:58377

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:58377
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.makeRequest(CollectionsAPIDistributedZkTest.java:400)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.testErrorHandling(CollectionsAPIDistributedZkTest.java:516)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test(CollectionsAPIDistributedZkTest.java:180)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 47 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:52014/_/v/c8n_1x3_lf_shard1_replica1]

Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: No live SolrServers available to handle this request:[http://127.0.0.1:52014/_/v/c8n_1x3_lf_shard1_replica1]
	at __randomizedtesting.SeedInfo.seed([CAAE2B1DF4390888:42FA14C75AC56570]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:755)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1161)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1050)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:992)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.sendDocsWithRetry(AbstractFullDistribZkTestBase.java:753)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.sendDocsWithRetry(AbstractFullDistribZkTestBase.java:741)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:178)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:57)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:52014/_/v/c8n_1x3_lf_shard1_replica1]
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:394)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:730)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more
Caused by: org.apache.solr.client.solrj.SolrServerException: Server refused connection at: http://127.0.0.1:52014/_/v/c8n_1x3_lf_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:615)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:415)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:383)
	... 6 more
Caused by: org.apache.http.conn.HttpHostConnectException: Connect to 127.0.0.1:52014 [/127.0.0.1] failed: Connection refused
	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:151)
	at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
	at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 10 more
Caused by: java.net.ConnectException: Connection refused
	at java.net.PlainSocketImpl.socketConnect(Native Method)
	at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
	at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
	at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
	at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
	at java.net.Socket.connect(Socket.java:589)
	at org.apache.http.conn.socket.PlainConnectionSocketFactory.connectSocket(PlainConnectionSocketFactory.java:74)
	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:134)
	... 20 more


FAILED:  org.apache.solr.cloud.ForceLeaderTest.testReplicasInLIRNoLeader

Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:42892/_/kx/forceleader_test_collection_shard1_replica2]

Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: No live SolrServers available to handle this request:[http://127.0.0.1:42892/_/kx/forceleader_test_collection_shard1_replica2]
	at __randomizedtesting.SeedInfo.seed([CAAE2B1DF4390888:2C391FDDCDBBF1E9]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:755)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1161)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1050)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:992)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.sendDocsWithRetry(AbstractFullDistribZkTestBase.java:753)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.sendDocsWithRetry(AbstractFullDistribZkTestBase.java:741)
	at org.apache.solr.cloud.ForceLeaderTest.sendDoc(ForceLeaderTest.java:424)
	at org.apache.solr.cloud.ForceLeaderTest.testReplicasInLIRNoLeader(ForceLeaderTest.java:131)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:42892/_/kx/forceleader_test_collection_shard1_replica2]
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:394)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:730)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more
Caused by: org.apache.solr.client.solrj.SolrServerException: Server refused connection at: http://127.0.0.1:42892/_/kx/forceleader_test_collection_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:615)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:415)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:383)
	... 6 more
Caused by: org.apache.http.conn.HttpHostConnectException: Connect to 127.0.0.1:42892 [/127.0.0.1] failed: Connection refused
	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:151)
	at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
	at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 10 more
Caused by: java.net.ConnectException: Connection refused
	at java.net.PlainSocketImpl.socketConnect(Native Method)
	at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
	at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
	at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
	at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
	at java.net.Socket.connect(Socket.java:589)
	at org.apache.http.conn.socket.PlainConnectionSocketFactory.connectSocket(PlainConnectionSocketFactory.java:74)
	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:134)
	... 20 more


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestStressReplication

Error Message:
timed out waiting for collection1 startAt time to exceed: Thu Sep 01 04:54:40 CHUT 2016

Stack Trace:
java.lang.AssertionError: timed out waiting for collection1 startAt time to exceed: Thu Sep 01 04:54:40 CHUT 2016
	at __randomizedtesting.SeedInfo.seed([CAAE2B1DF4390888:11052BDBF111613B]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.handler.TestReplicationHandler.watchCoreStartAt(TestReplicationHandler.java:1508)
	at org.apache.solr.handler.TestReplicationHandler.doTestStressReplication(TestReplicationHandler.java:858)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard1

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard1
	at __randomizedtesting.SeedInfo.seed([CAAE2B1DF4390888:96B3CC9C3A19BCBE]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart(CdcrReplicationDistributedZkTest.java:235)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
	at __randomizedtesting.SeedInfo.seed([CAAE2B1DF4390888]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest:     1) Thread[id=22062, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=22057, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=22082, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=22052, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=22067, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=22037, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=22072, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=22042, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=22047, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=22077, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 
   1) Thread[id=22062, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=22057, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=22082, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=22052, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=22067, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=22037, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=22072, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=22042, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=22047, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=22077, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([CAAE2B1DF4390888]:0)




Build Log:
[...truncated 11613 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_CAAE2B1DF4390888-001/init-core-data-001
   [junit4]   2> 51675 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[CAAE2B1DF4390888]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (true) and clientAuth (true) via: @org.apache.solr.util.RandomizeSSL(reason=, ssl=NaN, value=NaN, clientAuth=NaN)
   [junit4]   2> 51923 INFO  (SUITE-CollectionsAPIDistributedZkTest-seed#[CAAE2B1DF4390888]-worker) [    ] o.a.s.BaseDistributedSearchTestCase Setting hostContext system property: /
   [junit4]   2> 51949 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.ZkTestServer STARTING ZK TEST SERVER
   [junit4]   2> 51961 INFO  (Thread-35) [    ] o.a.s.c.ZkTestServer client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 51961 INFO  (Thread-35) [    ] o.a.s.c.ZkTestServer Starting server
   [junit4]   2> 52132 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.ZkTestServer start zk server on port:44563
   [junit4]   2> 52133 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 52149 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 52160 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn Exception causing close of session 0x0 due to java.io.IOException: ZooKeeperServer not running
   [junit4]   2> 53805 INFO  (zkCallback-32-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@7dc2bc1b name:ZooKeeperConnection Watcher:127.0.0.1:44563 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 53805 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 53805 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 53805 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /solr
   [junit4]   2> 53813 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 53835 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 53839 INFO  (zkCallback-33-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@42eff37a name:ZooKeeperConnection Watcher:127.0.0.1:44563/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 53839 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 53840 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 53840 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1
   [junit4]   2> 53849 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/collection1/shards
   [junit4]   2> 53859 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection
   [junit4]   2> 53861 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /collections/control_collection/shards
   [junit4]   2> 53862 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 53863 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 53867 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 53867 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/schema.xml
   [junit4]   2> 53869 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 53869 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 53885 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 53885 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 53888 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 53888 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/protwords.txt
   [junit4]   2> 53890 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 53890 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/currency.xml
   [junit4]   2> 53892 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 53892 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 53895 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 53895 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 53897 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 53897 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 53902 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 53902 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 53904 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.AbstractZkTestCase put /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 53904 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 53926 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 53960 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 53966 INFO  (zkCallback-35-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@6852de2 name:ZooKeeperConnection Watcher:127.0.0.1:44563/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 53966 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 53966 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 53968 WARN  (NIOServerCxn.Factory:0.0.0.0/0.0.0.0:0) [    ] o.a.z.s.NIOServerCnxn caught end of stream exception
   [junit4]   2> EndOfStreamException: Unable to read additional data from client sessionid 0x156e1d293e80002, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 54163 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_CAAE2B1DF4390888-001/control-001/cores/collection1
   [junit4]   2> 54247 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 54365 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@4fa6eb77{/,null,AVAILABLE}
   [junit4]   2> 55137 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.e.j.s.ServerConnector Started ServerConnector@44aab337{SSL,[ssl, http/1.1]}{127.0.0.1:49385}
   [junit4]   2> 55137 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.e.j.s.Server Started @59695ms
   [junit4]   2> 55137 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_CAAE2B1DF4390888-001/tempDir-001/control/data, hostContext=/, hostPort=49385, coreRootDirectory=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_CAAE2B1DF4390888-001/control-001/cores}
   [junit4]   2> 55140 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 55144 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_CAAE2B1DF4390888-001/control-001'
   [junit4]   2> 55144 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 55144 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 55146 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 55163 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 55187 INFO  (zkCallback-36-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@21fb6de1 name:ZooKeeperConnection Watcher:127.0.0.1:44563/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 55187 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 55187 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 55208 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.s.SolrDispatchFilter Loading solr.xml from SolrHome (not found in ZooKeeper)
   [junit4]   2> 55209 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_CAAE2B1DF4390888-001/control-001/solr.xml
   [junit4]   2> 55221 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_CAAE2B1DF4390888-001/control-001/cores
   [junit4]   2> 55221 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.CoreContainer New CoreContainer 685098092
   [junit4]   2> 55221 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_CAAE2B1DF4390888-001/control-001]
   [junit4]   2> 55221 WARN  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.CoreContainer Couldn't add files from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_CAAE2B1DF4390888-001/control-001/lib to classpath: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_CAAE2B1DF4390888-001/control-001/lib
   [junit4]   2> 55221 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,
   [junit4]   2> 55265 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.u.UpdateShardHandler Created UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&maxConnections=100000&maxConnectionsPerHost=100000
   [junit4]   2> 55265 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 55265 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 55265 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.ZkContainer Zookeeper client=127.0.0.1:44563/solr
   [junit4]   2> 55265 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.ZkController zkHost includes chroot
   [junit4]   2> 55266 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient Using default ZkCredentialsProvider
   [junit4]   2> 55282 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 55300 INFO  (zkCallback-39-thread-1) [    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@72b01c8a name:ZooKeeperConnection Watcher:127.0.0.1:44563 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 55302 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 55302 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [    ] o.a.s.c.c.SolrZkClient Using default ZkACLProvider
   [junit4]   2> 55310 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.ZkController Added new OnReconnect listener org.apache.solr.cloud.ZkController$$Lambda$74/1552792430@56cbae7e
   [junit4]   2> 55328 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.ConnectionManager Waiting for client to connect to ZooKeeper
   [junit4]   2> 55356 INFO  (zkCallback-40-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_    ] o.a.s.c.c.ConnectionManager Watcher org.apache.solr.common.cloud.ConnectionManager@621bdbb7 name:ZooKeeperConnection Watcher:127.0.0.1:44563/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 55358 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.ConnectionManager Client is connected to ZooKeeper
   [junit4]   2> 55360 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue
   [junit4]   2> 55367 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-queue-work
   [junit4]   2> 55370 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-running
   [junit4]   2> 55373 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-completed
   [junit4]   2> 55375 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/collection-map-failure
   [junit4]   2> 55376 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes
   [junit4]   2> 55386 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /aliases.json
   [junit4]   2> 55388 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /clusterstate.json
   [junit4]   2> 55395 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /security.json
   [junit4]   2> 55396 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.ZkStateReader Updating cluster state from ZooKeeper... 
   [junit4]   2> 55397 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.ZkStateReader Loaded cluster properties: {urlScheme=https}
   [junit4]   2> 55397 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 55401 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect
   [junit4]   2> 55402 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/election
   [junit4]   2> 55403 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.Overseer Overseer (id=null) closing
   [junit4]   2> 55404 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.LeaderElector Joined leadership election with path: /overseer_elect/election/96512736575881221-127.0.0.1:49385_-n_0000000000
   [junit4]   2> 55404 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.OverseerElectionContext I am going to be the leader 127.0.0.1:49385_
   [junit4]   2> 55404 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /overseer_elect/leader
   [junit4]   2> 55406 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.Overseer Overseer (id=96512736575881221-127.0.0.1:49385_-n_0000000000) starting
   [junit4]   2> 55408 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /overseer/queue-work
   [junit4]   2> 55414 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.OverseerAutoReplicaFailoverThread Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=10000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 55425 INFO  (OverseerStateUpdate-96512736575881221-127.0.0.1:49385_-n_0000000000) [n:127.0.0.1:49385_    ] o.a.s.c.Overseer Starting to work on the main queue
   [junit4]   2> 55436 INFO  (OverseerStateUpdate-96512736575881221-127.0.0.1:49385_-n_0000000000) [n:127.0.0.1:49385_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (0)
   [junit4]   2> 55444 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.ZkController Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49385_
   [junit4]   2> 55444 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.c.SolrZkClient makePath: /live_nodes/127.0.0.1:49385_
   [junit4]   2> 55446 INFO  (zkCallback-40-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_    ] o.a.s.c.c.ZkStateReader A live node change: [WatchedEvent state:SyncConnected type:NodeChildrenChanged path:/live_nodes], has occurred - updating... (live nodes size: [0])
   [junit4]   2> 55446 INFO  (OverseerCollectionConfigSetProcessor-96512736575881221-127.0.0.1:49385_-n_0000000000) [n:127.0.0.1:49385_    ] o.a.s.c.OverseerTaskProcessor Process current queue of overseer operations
   [junit4]   2> 55451 INFO  (zkCallback-40-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_    ] o.a.s.c.c.ZkStateReader Updated live nodes from ZooKeeper... (0) -> (1)
   [junit4]   2> 55481 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 55481 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 55482 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_CAAE2B1DF4390888-001/control-001/cores
   [junit4]   2> 55483 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection1, config=solrconfig.xml, transient=false, schema=schema.xml, loadOnStartup=true, configSetProperties=configsetprops.json, collection=control_collection, coreNodeName=, dataDir=data/, shard=}
   [junit4]   2> 55484 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.CorePropertiesLocator Found core collection1 in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/../../../../../../../../../../../x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_CAAE2B1DF4390888-001/control-001/cores/collection1
   [junit4]   2> 55484 INFO  (TEST-CollectionsAPIDistributedZkTest.test-seed#[CAAE2B1DF4390888]) [n:127.0.0.1:49385_    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions
   [junit4]   2> 55580 INFO  (coreLoadExecutor-105-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_ c:control_collection   x:collection1] o.a.s.c.ZkController publishing state=down
   [junit4]   2> 55580 INFO  (coreLoadExecutor-105-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_ c:control_collection   x:collection1] o.a.s.c.ZkController numShards not found on descriptor - reading it from system property
   [junit4]   2> 55582 INFO  (coreLoadExecutor-105-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_ c:control_collection   x:collection1] o.a.s.c.ZkController Collection control_collection not visible yet, but flagging it so a watch is registered when it becomes visible
   [junit4]   2> 55583 INFO  (OverseerStateUpdate-96512736575881221-127.0.0.1:49385_-n_0000000000) [n:127.0.0.1:49385_    ] o.a.s.c.Overseer processMessage: queueSize: 1, message = {
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"https://127.0.0.1:49385",
   [junit4]   2>   "node_name":"127.0.0.1:49385_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":null,
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "operation":"state"} current state version: 0
   [junit4]   2> 55583 INFO  (OverseerStateUpdate-96512736575881221-127.0.0.1:49385_-n_0000000000) [n:127.0.0.1:49385_    ] o.a.s.c.o.ReplicaMutator Update state numShards=1 message={
   [junit4]   2>   "core":"collection1",
   [junit4]   2>   "roles":null,
   [junit4]   2>   "base_url":"https://127.0.0.1:49385",
   [junit4]   2>   "node_name":"127.0.0.1:49385_",
   [junit4]   2>   "numShards":"1",
   [junit4]   2>   "state":"down",
   [junit4]   2>   "shard":null,
   [junit4]   2>   "collection":"control_collection",
   [junit4]   2>   "operation":"state"}
   [junit4]   2> 55584 INFO  (OverseerStateUpdate-96512736575881221-127.0.0.1:49385_-n_0000000000) [n:127.0.0.1:49385_    ] o.a.s.c.o.ClusterStateMutator building a new cName: control_collection
   [junit4]   2> 55585 INFO  (coreLoadExecutor-105-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_ c:control_collection   x:collection1] o.a.s.c.c.ZkStateReader Deleting data for [control_collection]
   [junit4]   2> 55586 INFO  (OverseerStateUpdate-96512736575881221-127.0.0.1:49385_-n_0000000000) [n:127.0.0.1:49385_    ] o.a.s.c.o.ReplicaMutator Assigning new node to shard shard=shard1
   [junit4]   2> 55588 INFO  (zkCallback-40-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_    ] o.a.s.c.c.ZkStateReader A cluster state change: [WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json], has occurred - updating... (live nodes size: [1])
   [junit4]   2> 55611 INFO  (coreLoadExecutor-105-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_ c:control_collection   x:collection1] o.a.s.c.ZkController look for our core node name
   [junit4]   2> 55611 INFO  (coreLoadExecutor-105-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_ c:control_collection   x:collection1] o.a.s.c.ZkController waiting to find shard id in clusterstate for collection1
   [junit4]   2> 55611 INFO  (coreLoadExecutor-105-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_ c:control_collection   x:collection1] o.a.s.c.ZkController Check for collection zkNode:control_collection
   [junit4]   2> 55611 INFO  (coreLoadExecutor-105-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_ c:control_collection   x:collection1] o.a.s.c.ZkController Collection zkNode exists
   [junit4]   2> 55611 INFO  (coreLoadExecutor-105-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_ c:control_collection   x:collection1] o.a.s.c.c.ZkStateReader Load collection config from: [/collections/control_collection]
   [junit4]   2> 55612 INFO  (coreLoadExecutor-105-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_ c:control_collection   x:collection1] o.a.s.c.c.ZkStateReader path=[/collections/control_collection] [configName]=[conf1] specified config exists in ZooKeeper
   [junit4]   2> 55612 INFO  (coreLoadExecutor-105-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_ c:control_collection   x:collection1] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J1/temp/solr.cloud.CollectionsAPIDistributedZkTest_CAAE2B1DF4390888-001/control-001/cores/collection1'
   [junit4]   2> 55612 INFO  (coreLoadExecutor-105-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_ c:control_collection   x:collection1] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 55612 INFO  (coreLoadExecutor-105-thread-1-processing-n:127.0.0.1:49385_) [n:127.0.0.1:49385_ c:control_collection   x:collection1] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2

[...truncated too long message...]

est(LBHttpSolrClient.java:394)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:730)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
   [junit4]    > 	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
   [junit4]    > 	... 1 more
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: Server refused connection at: http://127.0.0.1:42892/_/kx/forceleader_test_collection_shard1_replica2
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:615)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:415)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:383)
   [junit4]    > 	... 6 more
   [junit4]    > Caused by: org.apache.http.conn.HttpHostConnectException: Connect to 127.0.0.1:42892 [/127.0.0.1] failed: Connection refused
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:151)
   [junit4]    > 	at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
   [junit4]    > 	at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
   [junit4]    > 	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
   [junit4]    > 	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
   [junit4]    > 	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
   [junit4]    > 	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
   [junit4]    > 	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
   [junit4]    > 	... 10 more
   [junit4]    > Caused by: java.net.ConnectException: Connection refused
   [junit4]    > 	at java.net.PlainSocketImpl.socketConnect(Native Method)
   [junit4]    > 	at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
   [junit4]    > 	at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
   [junit4]    > 	at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
   [junit4]    > 	at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
   [junit4]    > 	at java.net.Socket.connect(Socket.java:589)
   [junit4]    > 	at org.apache.http.conn.socket.PlainConnectionSocketFactory.connectSocket(PlainConnectionSocketFactory.java:74)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:134)
   [junit4]    > 	... 20 more
   [junit4]   2> 3357233 INFO  (SUITE-ForceLeaderTest-seed#[CAAE2B1DF4390888]-worker) [    ] o.a.s.SolrTestCaseJ4 ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.cloud.ForceLeaderTest_CAAE2B1DF4390888-001
   [junit4]   2> Aug 31, 2016 7:13:15 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {range_facet_l_dv=Lucene50(blocksize=128), _version_=PostingsFormat(name=Memory doPackFST= false), multiDefault=PostingsFormat(name=Direct), a_t=PostingsFormat(name=Direct), intDefault=PostingsFormat(name=Memory doPackFST= false), id=Lucene50(blocksize=128), range_facet_i_dv=PostingsFormat(name=Direct), text=PostingsFormat(name=LuceneVarGapFixedInterval), range_facet_l=PostingsFormat(name=Direct), timestamp=PostingsFormat(name=Direct)}, docValues:{range_facet_l_dv=DocValuesFormat(name=Direct), range_facet_i_dv=DocValuesFormat(name=Lucene54), timestamp=DocValuesFormat(name=Lucene54)}, maxPointsInLeafNode=1438, maxMBSortInHeap=7.336041322335511, sim=ClassicSimilarity, locale=nl-BE, timezone=Africa/Bamako
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=312237992,total=513277952
   [junit4]   2> NOTE: All tests run in this JVM: [TestFoldingMultitermQuery, TestSubQueryTransformerDistrib, BlockJoinFacetRandomTest, CleanupOldIndexTest, CollectionsAPISolrJTest, StandardRequestHandlerTest, TestAnalyzedSuggestions, FileBasedSpellCheckerTest, BasicDistributedZkTest, TestDistribDocBasedVersion, TestSQLHandler, UUIDUpdateProcessorFallbackTest, XsltUpdateRequestHandlerTest, TestRawTransformer, AsyncCallRequestStatusResponseTest, DateRangeFieldTest, DocumentAnalysisRequestHandlerTest, TestDocTermOrds, CacheHeaderTest, DateFieldTest, TestSolrXml, HdfsWriteToMultipleCollectionsTest, TestLegacyNumericRangeQueryBuilder, TestReplicationHandlerBackup, BigEndianAscendingWordDeserializerTest, SolrCloudExampleTest, TestDocSet, JSONWriterTest, TestHighlightDedupGrouping, ConnectionReuseTest, TestTrieFacet, TestDynamicFieldResource, TestPhraseSuggestions, TestSolr4Spatial, TestNumericTerms32, TestManagedSchema, EchoParamsTest, DataDrivenBlockJoinTest, OutputWriterTest, TestRemoteStreaming, QueryElevationComponentTest, CursorPagingTest, WrapperMergePolicyFactoryTest, TestRandomCollapseQParserPlugin, ZkStateReaderTest, ReturnFieldsTest, DeleteInactiveReplicaTest, TestPKIAuthenticationPlugin, TestRecoveryHdfs, TestCopyFieldCollectionResource, IndexSchemaRuntimeFieldTest, TestTolerantUpdateProcessorCloud, SolrRequestParserTest, TestFieldCacheSanityChecker, TestBadConfig, HttpPartitionTest, TestValueSourceCache, TestComponentsName, TestStressLucene, ZkNodePropsTest, QueryResultKeyTest, ResponseHeaderTest, TestUseDocValuesAsStored, OutOfBoxZkACLAndCredentialsProvidersTest, AtomicUpdatesTest, ExplicitHLLTest, DefaultValueUpdateProcessorTest, TestJsonFacets, TriLevelCompositeIdRoutingTest, CloudMLTQParserTest, DocExpirationUpdateProcessorFactoryTest, CollectionStateFormat2Test, CopyFieldTest, AnalysisAfterCoreReloadTest, TestSolrCloudWithSecureImpersonation, DistributedDebugComponentTest, HdfsThreadLeakTest, TestRealTimeGet, TestFaceting, TestDFISimilarityFactory, AddBlockUpdateTest, TestOnReconnectListenerSupport, TestMergePolicyConfig, DistributedFacetPivotLongTailTest, SynonymTokenizerTest, PolyFieldTest, SuggestComponentContextFilterQueryTest, TestRandomFaceting, RAMDirectoryFactoryTest, JavabinLoaderTest, HdfsRecoveryZkTest, SaslZkACLProviderTest, QueryEqualityTest, ExternalFileFieldSortTest, TestHdfsCloudBackupRestore, ResourceLoaderTest, TestChildDocTransformer, PathHierarchyTokenizerFactoryTest, TestDynamicLoading, TestNoOpRegenerator, PreAnalyzedUpdateProcessorTest, TestSchemaResource, SyncSliceTest, TestLazyCores, ChaosMonkeySafeLeaderTest, UnloadDistributedZkTest, OverseerTest, LeaderElectionIntegrationTest, ShardRoutingTest, BasicZkTest, TestReplicationHandler, TestDistributedGrouping, TermVectorComponentDistributedTest, TestJoin, TestStressVersions, TestCoreContainer, SolrCoreTest, StatsComponentTest, TestFiltering, SimplePostToolTest, TestCoreDiscovery, TestExtendedDismaxParser, DocValuesTest, DisMaxRequestHandlerTest, DirectSolrSpellCheckerTest, TestReversedWildcardFilterFactory, TestQueryTypes, TermsComponentTest, RequestHandlersTest, TermVectorComponentTest, TestArbitraryIndexDir, LoggingHandlerTest, TestJmxIntegration, BinaryUpdateRequestHandlerTest, TestPHPSerializedResponseWriter, SearchHandlerTest, UpdateParamsTest, TestSolrIndexConfig, BadComponentTest, TestStressRecovery, SolrTestCaseJ4Test, TestCursorMarkWithoutUniqueKey, TestSimpleTrackingShardHandler, BaseCdcrDistributedZkTest, CdcrBootstrapTest, CloudExitableDirectoryReaderTest, CollectionsAPIAsyncDistributedZkTest, ConcurrentDeleteAndCreateCollectionTest, DeleteShardTest, DistribCursorPagingTest, ForceLeaderTest]
   [junit4] Completed [378/632 (6!)] on J0 in 115.74s, 3 tests, 1 error, 1 skipped <<< FAILURES!

[...truncated 828 lines...]
   [junit4] JVM J0: stdout was not empty, see: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/temp/junit4-J0-20160831_181712_490.sysout
   [junit4] >>> JVM J0 emitted unexpected output (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: GC overhead limit exceeded
   [junit4] Dumping heap to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/heapdumps/java_pid13708.hprof ...
   [junit4] Heap dump file created [533401820 bytes in 2.740 secs]
   [junit4] <<< JVM J0: EOF ----

[...truncated 11041 lines...]
BUILD FAILED
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:771: The following error occurred while executing this line:
/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/build.xml:715: Some of the tests produced a heap dump, but did not fail. Maybe a suppressed OutOfMemoryError? Dumps created:
* java_pid13708.hprof

Total time: 342 minutes 11 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Recording test results
Email was triggered for: Failure - Any
Sending email for trigger: Failure - Any




[JENKINS] Lucene-Solr-NightlyTests-master - Build # 1108 - Still unstable

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-master/1108/

10 tests failed.
FAILED:  org.apache.lucene.search.TestSearcherManager.testConcurrentIndexCloseSearchAndRefresh

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([B1F0697F7D82E83F]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.search.TestSearcherManager

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([B1F0697F7D82E83F]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.search.TestSearcherManager

Error Message:
Captured an uncaught exception in thread: Thread[id=4271, name=Thread-3444, state=RUNNABLE, group=TGRP-TestSearcherManager]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=4271, name=Thread-3444, state=RUNNABLE, group=TGRP-TestSearcherManager]
Caused by: java.lang.IllegalArgumentException: Document contains at least one immense term in field="body" (whose UTF8 encoding is longer than the max length 32766), all of which were skipped.  Please correct the analyzer to not produce such terms.  The prefix of the first immense term is: '[125, 125, 123, 123, 123, 123, 123, 115, 117, 98, 115, 116, 99, 124, 125, 125, 125, 123, 123, 123, 49, 125, 125, 125, 124, 123, 123, 123, 112, 49]...', original message: bytes can be at most 32766 in length; got 94384
	at __randomizedtesting.SeedInfo.seed([B1F0697F7D82E83F]:0)
	at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:772)
	at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:417)
	at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:373)
	at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:231)
	at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:478)
	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1562)
	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
	at org.apache.lucene.search.TestSearcherManager$8.run(TestSearcherManager.java:557)
	Suppressed: java.lang.IllegalStateException: close() called in wrong state: INCREMENT
		at org.apache.lucene.analysis.MockTokenizer.fail(MockTokenizer.java:125)
		at org.apache.lucene.analysis.MockTokenizer.close(MockTokenizer.java:292)
		at org.apache.lucene.analysis.TokenFilter.close(TokenFilter.java:58)
		at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
		... 7 more
Caused by: org.apache.lucene.util.BytesRefHash$MaxBytesLengthExceededException: bytes can be at most 32766 in length; got 94384
	at org.apache.lucene.util.BytesRefHash.add(BytesRefHash.java:263)
	at org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:149)
	at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:762)
	... 7 more


FAILED:  org.apache.solr.cloud.ForceLeaderTest.testReplicasInLIRNoLeader

Error Message:
No live SolrServers available to handle this request:[http://127.0.0.1:33079/forceleader_test_collection_shard1_replica2]

Stack Trace:
org.apache.solr.client.solrj.impl.CloudSolrClient$RouteException: No live SolrServers available to handle this request:[http://127.0.0.1:33079/forceleader_test_collection_shard1_replica2]
	at __randomizedtesting.SeedInfo.seed([89FA53BCCCDAFAFA:6F6D677CF558039B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:755)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1161)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1050)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:992)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.sendDocsWithRetry(AbstractFullDistribZkTestBase.java:753)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.sendDocsWithRetry(AbstractFullDistribZkTestBase.java:741)
	at org.apache.solr.cloud.ForceLeaderTest.sendDoc(ForceLeaderTest.java:424)
	at org.apache.solr.cloud.ForceLeaderTest.testReplicasInLIRNoLeader(ForceLeaderTest.java:131)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:33079/forceleader_test_collection_shard1_replica2]
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:394)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.lambda$directUpdate$0(CloudSolrClient.java:730)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.solr.common.util.ExecutorUtil$MDCAwareThreadPoolExecutor.lambda$execute$0(ExecutorUtil.java:229)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
	... 1 more
Caused by: org.apache.solr.client.solrj.SolrServerException: Server refused connection at: http://127.0.0.1:33079/forceleader_test_collection_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:615)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:415)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:383)
	... 6 more
Caused by: org.apache.http.conn.HttpHostConnectException: Connect to 127.0.0.1:33079 [/127.0.0.1] failed: Connection refused
	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:151)
	at org.apache.http.impl.conn.PoolingHttpClientConnectionManager.connect(PoolingHttpClientConnectionManager.java:353)
	at org.apache.http.impl.execchain.MainClientExec.establishRoute(MainClientExec.java:380)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:236)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 10 more
Caused by: java.net.ConnectException: Connection refused
	at java.net.PlainSocketImpl.socketConnect(Native Method)
	at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
	at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
	at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
	at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
	at java.net.Socket.connect(Socket.java:589)
	at org.apache.http.conn.socket.PlainConnectionSocketFactory.connectSocket(PlainConnectionSocketFactory.java:74)
	at org.apache.http.impl.conn.DefaultHttpClientConnectionOperator.connect(DefaultHttpClientConnectionOperator.java:134)
	... 20 more


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestStressReplication

Error Message:
timed out waiting for collection1 startAt time to exceed: Tue Aug 30 00:24:47 HDT 2016

Stack Trace:
java.lang.AssertionError: timed out waiting for collection1 startAt time to exceed: Tue Aug 30 00:24:47 HDT 2016
	at __randomizedtesting.SeedInfo.seed([89FA53BCCCDAFAFA:5251537AC9F29349]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.handler.TestReplicationHandler.watchCoreStartAt(TestReplicationHandler.java:1508)
	at org.apache.solr.handler.TestReplicationHandler.doTestStressReplication(TestReplicationHandler.java:858)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.security.BasicAuthIntegrationTest.testBasics

Error Message:
IOException occured when talking to server at: http://127.0.0.1:34082/solr

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:34082/solr
	at __randomizedtesting.SeedInfo.seed([89FA53BCCCDAFAFA:B422FD90F434A48A]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:622)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:415)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:367)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1280)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1050)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:992)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.security.BasicAuthIntegrationTest.doExtraTests(BasicAuthIntegrationTest.java:116)
	at org.apache.solr.cloud.TestMiniSolrCloudClusterBase.testCollectionCreateSearchDelete(TestMiniSolrCloudClusterBase.java:196)
	at org.apache.solr.cloud.TestMiniSolrCloudClusterBase.testBasics(TestMiniSolrCloudClusterBase.java:79)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:34082 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 50 more


FAILED:  org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart

Error Message:
Timeout waiting for CDCR replication to complete @source_collection:shard1

Stack Trace:
java.lang.RuntimeException: Timeout waiting for CDCR replication to complete @source_collection:shard1
	at __randomizedtesting.SeedInfo.seed([89FA53BCCCDAFAFA:D5E7B43D02FA4ECC]:0)
	at org.apache.solr.cloud.BaseCdcrDistributedZkTest.waitForReplicationToComplete(BaseCdcrDistributedZkTest.java:794)
	at org.apache.solr.cloud.CdcrReplicationDistributedZkTest.testReplicationAfterRestart(CdcrReplicationDistributedZkTest.java:264)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:985)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:960)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]

Stack Trace:
java.lang.AssertionError: ObjectTracker found 10 object(s) that were not released!!! [InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient, InternalHttpClient]
	at __randomizedtesting.SeedInfo.seed([89FA53BCCCDAFAFA]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNull(Assert.java:551)
	at org.apache.solr.SolrTestCaseJ4.teardownTestCases(SolrTestCaseJ4.java:257)
	at sun.reflect.GeneratedMethodAccessor94.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:498)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:834)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest

Error Message:
10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest:     1) Thread[id=92689, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=92709, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    3) Thread[id=92729, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    4) Thread[id=92694, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    5) Thread[id=92714, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    6) Thread[id=92699, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    7) Thread[id=92724, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    8) Thread[id=92704, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)    9) Thread[id=92734, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)   10) Thread[id=92719, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]         at java.lang.Thread.sleep(Native Method)         at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 10 threads leaked from SUITE scope at org.apache.solr.cloud.ConcurrentDeleteAndCreateCollectionTest: 
   1) Thread[id=92689, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=92709, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   3) Thread[id=92729, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   4) Thread[id=92694, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   5) Thread[id=92714, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   6) Thread[id=92699, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   7) Thread[id=92724, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   8) Thread[id=92704, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
   9) Thread[id=92734, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
  10) Thread[id=92719, name=Connection evictor, state=TIMED_WAITING, group=TGRP-ConcurrentDeleteAndCreateCollectionTest]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([89FA53BCCCDAFAFA]:0)


FAILED:  org.apache.solr.cloud.hdfs.HdfsCollectionsAPIDistributedZkTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=188735, name=collection1, state=RUNNABLE, group=TGRP-HdfsCollectionsAPIDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=188735, name=collection1, state=RUNNABLE, group=TGRP-HdfsCollectionsAPIDistributedZkTest]
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:39426/_zfo
	at __randomizedtesting.SeedInfo.seed([89FA53BCCCDAFAFA]:0)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:998)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: https://127.0.0.1:39426/_zfo
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:619)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:261)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:250)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:415)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:367)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1280)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1050)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:992)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:995)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
	at java.net.SocketInputStream.read(SocketInputStream.java:170)
	at java.net.SocketInputStream.read(SocketInputStream.java:141)
	at sun.security.ssl.InputRecord.readFully(InputRecord.java:465)
	at sun.security.ssl.InputRecord.read(InputRecord.java:503)
	at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:973)
	at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:930)
	at sun.security.ssl.AppInputStream.read(AppInputStream.java:105)
	at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
	at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
	at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
	... 9 more




Build Log:
[...truncated 1845 lines...]
   [junit4] Suite: org.apache.lucene.search.TestSearcherManager
   [junit4]   2> серп. 30, 2016 5:05:57 PM com.carrotsearch.randomizedtesting.RandomizedRunner$QueueUncaughtExceptionsHandler uncaughtException
   [junit4]   2> WARNING: Uncaught exception in thread: Thread[Thread-3444,5,TGRP-TestSearcherManager]
   [junit4]   2> java.lang.IllegalArgumentException: Document contains at least one immense term in field="body" (whose UTF8 encoding is longer than the max length 32766), all of which were skipped.  Please correct the analyzer to not produce such terms.  The prefix of the first immense term is: '[125, 125, 123, 123, 123, 123, 123, 115, 117, 98, 115, 116, 99, 124, 125, 125, 125, 123, 123, 123, 49, 125, 125, 125, 124, 123, 123, 123, 112, 49]...', original message: bytes can be at most 32766 in length; got 94384
   [junit4]   2> 	at __randomizedtesting.SeedInfo.seed([B1F0697F7D82E83F]:0)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:772)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:417)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:373)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:231)
   [junit4]   2> 	at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:478)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1562)
   [junit4]   2> 	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
   [junit4]   2> 	at org.apache.lucene.search.TestSearcherManager$8.run(TestSearcherManager.java:557)
   [junit4]   2> 	Suppressed: java.lang.IllegalStateException: close() called in wrong state: INCREMENT
   [junit4]   2> 		at org.apache.lucene.analysis.MockTokenizer.fail(MockTokenizer.java:125)
   [junit4]   2> 		at org.apache.lucene.analysis.MockTokenizer.close(MockTokenizer.java:292)
   [junit4]   2> 		at org.apache.lucene.analysis.TokenFilter.close(TokenFilter.java:58)
   [junit4]   2> 		at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
   [junit4]   2> 		... 7 more
   [junit4]   2> Caused by: org.apache.lucene.util.BytesRefHash$MaxBytesLengthExceededException: bytes can be at most 32766 in length; got 94384
   [junit4]   2> 	at org.apache.lucene.util.BytesRefHash.add(BytesRefHash.java:263)
   [junit4]   2> 	at org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:149)
   [junit4]   2> 	at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:762)
   [junit4]   2> 	... 7 more
   [junit4]   2> 
   [junit4]   2> серп. 30, 2016 7:05:38 PM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.search.TestSearcherManager
   [junit4]   2>    1) Thread[id=4274, name=Thread-3447, state=RUNNABLE, group=TGRP-TestSearcherManager]
   [junit4]   2>         at java.lang.Thread.holdsLock(Native Method)
   [junit4]   2>         at org.apache.lucene.index.IndexFileDeleter.locked(IndexFileDeleter.java:117)
   [junit4]   2>         at org.apache.lucene.index.IndexFileDeleter.incRef(IndexFileDeleter.java:565)
   [junit4]   2>         at org.apache.lucene.index.IndexFileDeleter.incRef(IndexFileDeleter.java:553)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.incRefDeleter(IndexWriter.java:4920)
   [junit4]   2>         at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:117)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:460)
   [junit4]   2>         at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:103)
   [junit4]   2>         at org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:108)
   [junit4]   2>         at org.apache.lucene.search.SearcherManager.<init>(SearcherManager.java:76)
   [junit4]   2>         at org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:647)
   [junit4]   2>    2) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1249)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1323)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:601)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:450)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:243)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:354)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:10)
   [junit4]   2>    3) Thread[id=4273, name=Thread-3446, state=RUNNABLE, group=TGRP-TestSearcherManager]
   [junit4]   2>         at java.lang.Throwable.fillInStackTrace(Native Method)
   [junit4]   2>         at java.lang.Throwable.fillInStackTrace(Throwable.java:783)
   [junit4]   2>         at java.lang.Throwable.<init>(Throwable.java:265)
   [junit4]   2>         at java.lang.Exception.<init>(Exception.java:66)
   [junit4]   2>         at java.lang.RuntimeException.<init>(RuntimeException.java:62)
   [junit4]   2>         at java.lang.IllegalStateException.<init>(IllegalStateException.java:55)
   [junit4]   2>         at org.apache.lucene.store.AlreadyClosedException.<init>(AlreadyClosedException.java:26)
   [junit4]   2>         at org.apache.lucene.search.ReferenceManager.ensureOpen(ReferenceManager.java:54)
   [junit4]   2>         at org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:248)
   [junit4]   2>         at org.apache.lucene.search.TestSearcherManager$10.run(TestSearcherManager.java:617)
   [junit4]   2>    4) Thread[id=4269, name=SUITE-TestSearcherManager-seed#[B1F0697F7D82E83F], state=RUNNABLE, group=TGRP-TestSearcherManager]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1556)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:688)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$4.run(ThreadLeakControl.java:685)
   [junit4]   2>         at java.security.AccessController.doPrivileged(Native Method)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getStackTrace(ThreadLeakControl.java:685)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:701)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:681)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$1000(ThreadLeakControl.java:64)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:414)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:681)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:140)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$2.run(RandomizedRunner.java:591)
   [junit4]   2>    5) Thread[id=4270, name=TEST-TestSearcherManager.testConcurrentIndexCloseSearchAndRefresh-seed#[B1F0697F7D82E83F], state=WAITING, group=TGRP-TestSearcherManager]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1249)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1323)
   [junit4]   2>         at org.apache.lucene.search.TestSearcherManager.testConcurrentIndexCloseSearchAndRefresh(TestSearcherManager.java:670)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2>    6) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:47)
   [junit4]   2>    7) Thread[id=4272, name=Thread-3445, state=RUNNABLE, group=TGRP-TestSearcherManager]
   [junit4]   2>         at java.lang.Throwable.fillInStackTrace(Native Method)
   [junit4]   2>         at java.lang.Throwable.fillInStackTrace(Throwable.java:783)
   [junit4]   2>         at java.lang.Throwable.<init>(Throwable.java:265)
   [junit4]   2>         at java.lang.Exception.<init>(Exception.java:66)
   [junit4]   2>         at java.lang.RuntimeException.<init>(RuntimeException.java:62)
   [junit4]   2>         at java.lang.IllegalStateException.<init>(IllegalStateException.java:55)
   [junit4]   2>         at org.apache.lucene.store.AlreadyClosedException.<init>(AlreadyClosedException.java:26)
   [junit4]   2>         at org.apache.lucene.search.ReferenceManager.acquire(ReferenceManager.java:98)
   [junit4]   2>         at org.apache.lucene.search.TestSearcherManager$9.run(TestSearcherManager.java:588)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSearcherManager -Dtests.method=testConcurrentIndexCloseSearchAndRefresh -Dtests.seed=B1F0697F7D82E83F -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=uk-UA -Dtests.timezone=Pacific/Fakaofo -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   7200s J1 | TestSearcherManager.testConcurrentIndexCloseSearchAndRefresh <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([B1F0697F7D82E83F]:0)
   [junit4]   2> серп. 30, 2016 7:05:38 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 4 leaked thread(s).
   [junit4]   2> серп. 30, 2016 7:05:58 PM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 4 threads leaked from SUITE scope at org.apache.lucene.search.TestSearcherManager: 
   [junit4]   2>    1) Thread[id=4274, name=Thread-3447, state=RUNNABLE, group=TGRP-TestSearcherManager]
   [junit4]   2>         at java.lang.Thread.holdsLock(Native Method)
   [junit4]   2>         at org.apache.lucene.index.IndexFileDeleter.locked(IndexFileDeleter.java:117)
   [junit4]   2>         at org.apache.lucene.index.IndexFileDeleter.decRef(IndexFileDeleter.java:577)
   [junit4]   2>         at org.apache.lucene.index.IndexFileDeleter.decRef(IndexFileDeleter.java:647)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.decRefDeleter(IndexWriter.java:4932)
   [junit4]   2>         at org.apache.lucene.index.StandardDirectoryReader.doClose(StandardDirectoryReader.java:381)
   [junit4]   2>         at org.apache.lucene.index.IndexReader.decRef(IndexReader.java:251)
   [junit4]   2>         at org.apache.lucene.search.SearcherManager.decRef(SearcherManager.java:149)
   [junit4]   2>         at org.apache.lucene.search.SearcherManager.decRef(SearcherManager.java:58)
   [junit4]   2>         at org.apache.lucene.search.ReferenceManager.release(ReferenceManager.java:274)
   [junit4]   2>         at org.apache.lucene.search.ReferenceManager.swapReference(ReferenceManager.java:62)
   [junit4]   2>         at org.apache.lucene.search.ReferenceManager.close(ReferenceManager.java:146)
   [junit4]   2>         at org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:643)
   [junit4]   2>    2) Thread[id=4273, name=Thread-3446, state=RUNNABLE, group=TGRP-TestSearcherManager]
   [junit4]   2>         at java.lang.Throwable.fillInStackTrace(Native Method)
   [junit4]   2>         at java.lang.Throwable.fillInStackTrace(Throwable.java:783)
   [junit4]   2>         at java.lang.Throwable.<init>(Throwable.java:265)
   [junit4]   2>         at java.lang.Exception.<init>(Exception.java:66)
   [junit4]   2>         at java.lang.RuntimeException.<init>(RuntimeException.java:62)
   [junit4]   2>         at java.lang.IllegalStateException.<init>(IllegalStateException.java:55)
   [junit4]   2>         at org.apache.lucene.store.AlreadyClosedException.<init>(AlreadyClosedException.java:26)
   [junit4]   2>         at org.apache.lucene.search.ReferenceManager.ensureOpen(ReferenceManager.java:54)
   [junit4]   2>         at org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:248)
   [junit4]   2>         at org.apache.lucene.search.TestSearcherManager$10.run(TestSearcherManager.java:617)
   [junit4]   2>    3) Thread[id=4270, name=TEST-TestSearcherManager.testConcurrentIndexCloseSearchAndRefresh-seed#[B1F0697F7D82E83F], state=WAITING, group=TGRP-TestSearcherManager]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1249)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1323)
   [junit4]   2>         at org.apache.lucene.search.TestSearcherManager.testConcurrentIndexCloseSearchAndRefresh(TestSearcherManager.java:670)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:498)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1764)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:871)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$9.evaluate(RandomizedRunner.java:907)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$10.evaluate(RandomizedRunner.java:921)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:49)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:809)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:460)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:880)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:781)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:816)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:827)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:45)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:41)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:40)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:53)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:47)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:64)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:54)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:367)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2>    4) Thread[id=4272, name=Thread-3445, state=RUNNABLE, group=TGRP-TestSearcherManager]
   [junit4]   2>         at java.lang.Throwable.fillInStackTrace(Native Method)
   [junit4]   2>         at java.lang.Throwable.fillInStackTrace(Throwable.java:783)
   [junit4]   2>         at java.lang.Throwable.<init>(Throwable.java:265)
   [junit4]   2>         at java.lang.Exception.<init>(Exception.java:66)
   [junit4]   2>         at java.lang.RuntimeException.<init>(RuntimeException.java:62)
   [junit4]   2>         at java.lang.IllegalStateException.<init>(IllegalStateException.java:55)
   [junit4]   2>         at org.apache.lucene.store.AlreadyClosedException.<init>(AlreadyClosedException.java:26)
   [junit4]   2>         at org.apache.lucene.search.ReferenceManager.acquire(ReferenceManager.java:98)
   [junit4]   2>         at org.apache.lucene.search.TestSearcherManager$9.run(TestSearcherManager.java:588)
   [junit4]   2> серп. 30, 2016 7:05:58 PM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=4274, name=Thread-3447, state=RUNNABLE, group=TGRP-TestSearcherManager]
   [junit4]   2>    2) Thread[id=4273, name=Thread-3446, state=RUNNABLE, group=TGRP-TestSearcherManager]
   [junit4]   2>    3) Thread[id=4270, name=TEST-TestSearcherManager.testConcurrentIndexCloseSearchAndRefresh-seed#[B1F0697F7D82E83F], state=WAITING, group=TGRP-TestSearcherManager]
   [junit4]   2>    4) Thread[id=4272, name=Thread-3445, state=RUNNABLE, group=TGRP-TestSearcherManager]
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/lucene/build/core/test/J1/temp/lucene.search.TestSearcherManager_B1F0697F7D82E83F-001
   [junit4]   2> Aug 30, 2016 6:06:01 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=4274, name=Thread-3447, state=RUNNABLE, group=TGRP-TestSearcherManager]
   [junit4]   2>         at java.util.AbstractCollection.addAll(AbstractCollection.java:343)
   [junit4]   2>         at org.apache.lucene.index.SegmentCommitInfo.files(SegmentCommitInfo.java:234)
   [junit4]   2>         at org.apache.lucene.index.SegmentInfos.files(SegmentInfos.java:759)
   [junit4]   2>         at org.apache.lucene.index.IndexFileDeleter.decRef(IndexFileDeleter.java:647)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.decRefDeleter(IndexWriter.java:4932)
   [junit4]   2>         at org.apache.lucene.index.StandardDirectoryReader.doClose(StandardDirectoryReader.java:381)
   [junit4]   2>         at org.apache.lucene.index.IndexReader.decRef(IndexReader.java:251)
   [junit4]   2>         at org.apache.lucene.search.SearcherManager.decRef(SearcherManager.java:149)
   [junit4]   2>         at org.apache.lucene.search.SearcherManager.decRef(SearcherManager.java:58)
   [junit4]   2>         at org.apache.lucene.search.ReferenceManager.release(ReferenceManager.java:274)
   [junit4]   2>         at org.apache.lucene.search.ReferenceManager.swapReference(ReferenceManager.java:62)
   [junit4]   2>         at org.apache.lucene.search.ReferenceManager.close(ReferenceManager.java:146)
   [junit4]   2>         at org.apache.lucene.search.TestSearcherManager$11.run(TestSearcherManager.java:643)
   [junit4]   2>    2) Thread[id=4273, name=Thread-3446, state=RUNNABLE, group=TGRP-TestSearcherManager]
   [junit4]   2>         at java.lang.Throwable.fillInStackTrace(Native Method)
   [junit4]   2>         at java.lang.Throwable.fillInStackTrace(Throwable.java:783)
   [junit4]   2>         at java.lang.Throwable.<init>(Throwable.java:265)
   [junit4]   2>         at java.lang.Exception.<init>(Exception.java:66)
   [junit4]   2>         at java.lang.RuntimeException.<init>(RuntimeException.java:62)
   [junit4]   2>         at java.lang.IllegalStateException.<init>(IllegalStateException.java:55)
   [junit4]   2>         at org.apache.lucene.store.AlreadyClosedException.<init>(AlreadyClosedException.java:26)
   [junit4]   2>         at org.apache.lucene.search.ReferenceManager.ensureOpen(ReferenceManager.java:54)
   [junit4]   2>         at org.apache.lucene.search.ReferenceManager.maybeRefreshBlocking(ReferenceManager.java:248)
   [junit4]   2>         at org.apache.lucene.search.TestSearcherManager$10.run(TestSearcherManager.java:617)
   [junit4]   2>    3) Thread[id=4272, name=Thread-3445, state=RUNNABLE, group=TGRP-TestSearcherManager]
   [junit4]   2>         at java.lang.Throwable.fillInStackTrace(Native Method)
   [junit4]   2>         at java.lang.Throwable.fillInStackTrace(Throwable.java:783)
   [junit4]   2>         at java.lang.Throwable.<init>(Throwable.java:265)
   [junit4]   2>         at java.lang.Exception.<init>(Exception.java:66)
   [junit4]   2>         at java.lang.RuntimeException.<init>(RuntimeException.java:62)
   [junit4]   2>         at java.lang.IllegalStateException.<init>(IllegalStateException.java:55)
   [junit4]   2>         at org.apache.lucene.store.AlreadyClosedException.<init>(AlreadyClosedException.java:26)
   [junit4]   2>         at org.apache.lucene.search.ReferenceManager.acquire(ReferenceManager.java:98)
   [junit4]   2>         at org.apache.lucene.search.TestSearcherManager$9.run(TestSearcherManager.java:588)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {date=PostingsFormat(name=Asserting), docid=PostingsFormat(name=LuceneFixedGap), titleTokenized=PostingsFormat(name=MockRandom), body=PostingsFormat(name=Asserting), title=Lucene50(blocksize=128)}, docValues:{docid_intDV=DocValuesFormat(name=Lucene54), titleDV=DocValuesFormat(name=Asserting)}, maxPointsInLeafNode=451, maxMBSortInHeap=7.580362409278187, sim=ClassicSimilarity, locale=uk-UA, timezone=Pacific/Fakaofo
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=4,free=254920416,total=516947968
   [junit4]   2> NOTE: All tests run in this JVM: [TestForceMergeForever, TestVirtualMethod, TestLongPostings, TestLucene50TermVectorsFormat, TestStressIndexing2, TestLRUQueryCache, TestDirectoryReader, TestCheckIndex, TestScoreCachingWrappingScorer, TestNumericDocValuesUpdates, TestIndexWriterOnJRECrash, TestRegexpRandom, TestByteSlices, TestMutableValues, TestForTooMuchCloning, TestFieldValueQuery, TestByteArrayDataInput, TestWildcard, TestSearch, TestSortRandom, TestSumDocFreq, TestFastDecompressionMode, TestLucene50FieldInfoFormat, TestFieldCacheRewriteMethod, TestSentinelIntSet, TestBinaryTerms, TestDuelingCodecs, TestTermVectors, TestSortedSetDocValues, TestIndexSorting, TestSimpleExplanationsOfNonMatches, TestSpanNearQuery, TestDeterminism, TestAddIndexes, TestAllFilesHaveChecksumFooter, TestPagedBytes, TestNativeFSLockFactory, TestTieredMergePolicy, TestIndexInput, TestSimilarityProvider, TestIntroSelector, TestSubScorerFreqs, TestDocValuesRewriteMethod, TestIndexWriterOutOfFileDescriptors, TestFlex, TestIntBlockPool, TestSynonymQuery, Test2BSortedDocValuesOrds, TestBooleanQueryVisitSubscorers, TestPhraseQuery, TestGrowableByteArrayDataOutput, TestShardSearching, TestOperations, TestIndexWriterFromReader, TestTimeLimitingCollector, TestOmitTf, TestMultiFields, TestTransactionRollback, TestReqExclBulkScorer, TestMaxPosition, TestMixedDocValuesUpdates, TestDemo, TestIsCurrent, TestStressAdvance, TestNewestSegment, TestDateTools, TestIndexWriterOnDiskFull, TestHugeRamFile, TestBooleanScorer, TestNeedsScores, TestDocIDMerger, TestTrackingDirectoryWrapper, TestSearcherManager]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSearcherManager -Dtests.seed=B1F0697F7D82E83F -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/test-data/enwiki.random.lines.txt -Dtests.locale=uk-UA -Dtests.timezone=Pacific/Fakaofo -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J1 | TestSearcherManager (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([B1F0697F7D82E83F]:0)Throwable #2: com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=4271, name=Thread-3444, state=RUNNABLE, group=TGRP-TestSearcherManager]
   [junit4]    > Caused by: java.lang.IllegalArgumentException: Document contains at least one immense term in field="body" (whose UTF8 encoding is longer than the max length 32766), all of which were skipped.  Please correct the analyzer to not produce such terms.  The prefix of the first immense term is: '[125, 125, 123, 123, 123, 123, 123, 115, 117, 98, 115, 116, 99, 124, 125, 125, 125, 123, 123, 123, 49, 125, 125, 125, 124, 123, 123, 123, 112, 49]...', original message: bytes can be at most 32766 in length; got 94384
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([B1F0697F7D82E83F]:0)
   [junit4]    > 	at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:772)
   [junit4]    > 	at org.apache.lucene.index.DefaultIndexingChain.processField(DefaultIndexingChain.java:417)
   [junit4]    > 	at org.apache.lucene.index.DefaultIndexingChain.processDocument(DefaultIndexingChain.java:373)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriterPerThread.updateDocument(DocumentsWriterPerThread.java:231)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriter.updateDocument(DocumentsWriter.java:478)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1562)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1307)
   [junit4]    > 	at org.apache.lucene.search.TestSearcherManager$8.run(TestSearcherManager.java:557)
   [junit4]    > 	Suppressed: java.lang.IllegalStateException: close() called in wrong state: INCREMENT
   [junit4]    > 		at org.apache.lucene.analysis.MockTokenizer.fail(MockTokenizer.java:125)
   [junit4]    > 		at org.apache.lucene.analysis.MockTokenizer.close(MockTokenizer.java:292)
   [junit4]    > 		at org.apache.lucene.analysis.TokenFilter.close(TokenFilter.java:58)
   [junit4]    > 		at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:788)
   [junit4]    > 		... 7 more
   [junit4]    > Caused by: org.apache.lucene.util.BytesRefHash$MaxBytesLengthExceededException: bytes can be at most 32766 in length; got 94384
   [junit4]    > 	at org.apache.lucene.util.BytesRefHash.add(BytesRefHash.java:263)
   [junit4]    > 	at org.apache.lucene.index.TermsHashPerField.add(TermsHashPerField.java:149)
   [junit4]    > 	at org.apache.lucene.index.DefaultIndexingChain$PerField.invert(DefaultIndexingChain.java:762)
   [junit4]    > 	... 7 more
   [junit4] Completed [432/432 (1!)] on J1 in 7223.30s, 2 tests, 3 errors <<< FAILURES!

[...truncated 10371 lines...]
   [junit4] Suite: org.apache.solr.handler.TestReplicationHandler
   [junit4]   2> Creating dataDir: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_89FA53BCCCDAFAFA-001/init-core-data-001
   [junit4]   2> 2387540 INFO  (SUITE-TestReplicationHandler-seed#[89FA53BCCCDAFAFA]-worker) [    ] o.a.s.SolrTestCaseJ4 Randomized ssl (false) and clientAuth (false) via: @org.apache.solr.SolrTestCaseJ4$SuppressSSL(bugUrl=None)
   [junit4]   2> 2387548 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.a.s.SolrTestCaseJ4 ###Starting doTestIndexFetchWithMasterUrl
   [junit4]   2> 2387549 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.a.s.SolrTestCaseJ4 Writing core.properties file to /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_89FA53BCCCDAFAFA-001/solr-instance-001/collection1
   [junit4]   2> 2387562 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.e.j.s.Server jetty-9.3.8.v20160314
   [junit4]   2> 2387563 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.e.j.s.h.ContextHandler Started o.e.j.s.ServletContextHandler@7f96dcff{/solr,null,AVAILABLE}
   [junit4]   2> 2387565 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.e.j.s.ServerConnector Started ServerConnector@359e14cf{HTTP/1.1,[http/1.1]}{127.0.0.1:34106}
   [junit4]   2> 2387565 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.e.j.s.Server Started @2392989ms
   [junit4]   2> 2387565 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.a.s.c.s.e.JettySolrRunner Jetty properties: {solr.data.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_89FA53BCCCDAFAFA-001/solr-instance-001/collection1/data, hostContext=/solr, hostPort=34106}
   [junit4]   2> 2387565 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init(): sun.misc.Launcher$AppClassLoader@73d16e93
   [junit4]   2> 2387565 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_89FA53BCCCDAFAFA-001/solr-instance-001'
   [junit4]   2> 2387565 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2387565 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 2387565 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.a.s.c.SolrXmlConfig Loading container configuration from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_89FA53BCCCDAFAFA-001/solr-instance-001/solr.xml
   [junit4]   2> 2387577 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.a.s.c.CorePropertiesLocator Config-defined core root directory: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_89FA53BCCCDAFAFA-001/solr-instance-001/.
   [junit4]   2> 2387577 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.a.s.c.CoreContainer New CoreContainer 1705546384
   [junit4]   2> 2387577 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.a.s.c.CoreContainer Loading cores into CoreContainer [instanceDir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_89FA53BCCCDAFAFA-001/solr-instance-001]
   [junit4]   2> 2387578 WARN  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.a.s.c.CoreContainer Couldn't add files from /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_89FA53BCCCDAFAFA-001/solr-instance-001/lib to classpath: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_89FA53BCCCDAFAFA-001/solr-instance-001/lib
   [junit4]   2> 2387578 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.a.s.h.c.HttpShardHandlerFactory created with socketTimeout : 90000,urlScheme : ,connTimeout : 15000,maxConnectionsPerHost : 20,maxConnections : 10000,corePoolSize : 0,maximumPoolSize : 2147483647,maxThreadIdleTime : 5,sizeOfQueue : -1,fairnessPolicy : false,
   [junit4]   2> 2387580 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.a.s.u.UpdateShardHandler Created UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&maxConnections=100000&maxConnectionsPerHost=100000
   [junit4]   2> 2387580 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.a.s.l.LogWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 2387580 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.a.s.l.LogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2387580 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.a.s.c.CoreContainer Security conf doesn't exist. Skipping setup for authorization module.
   [junit4]   2> 2387580 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.a.s.c.CoreContainer No authentication plugin used.
   [junit4]   2> 2387581 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.a.s.c.CorePropertiesLocator Looking for core definitions underneath /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_89FA53BCCCDAFAFA-001/solr-instance-001/.
   [junit4]   2> 2387582 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.a.s.c.CoreDescriptor Created CoreDescriptor: {name=collection1, config=solrconfig.xml, loadOnStartup=true, schema=schema.xml, configSetProperties=configsetprops.json, transient=false, dataDir=data/}
   [junit4]   2> 2387582 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.a.s.c.CorePropertiesLocator Found core collection1 in /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_89FA53BCCCDAFAFA-001/solr-instance-001/./collection1
   [junit4]   2> 2387582 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.a.s.c.CorePropertiesLocator Found 1 core definitions
   [junit4]   2> 2387582 INFO  (coreLoadExecutor-4385-thread-1) [    ] o.a.s.c.SolrResourceLoader new SolrResourceLoader for directory: '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_89FA53BCCCDAFAFA-001/solr-instance-001/collection1'
   [junit4]   2> 2387583 INFO  (coreLoadExecutor-4385-thread-1) [    ] o.a.s.c.SolrResourceLoader JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 2387583 INFO  (coreLoadExecutor-4385-thread-1) [    ] o.a.s.c.SolrResourceLoader solr home defaulted to 'solr/' (could not find system property or JNDI)
   [junit4]   2> 2387587 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.a.s.s.SolrDispatchFilter user.dir=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0
   [junit4]   2> 2387587 INFO  (TEST-TestReplicationHandler.doTestIndexFetchWithMasterUrl-seed#[89FA53BCCCDAFAFA]) [    ] o.a.s.s.SolrDispatchFilter SolrDispatchFilter.init() done
   [junit4]   2> 2387596 INFO  (coreLoadExecutor-4385-thread-1) [    ] o.a.s.c.SolrConfig current version of requestparams : -1
   [junit4]   2> 2387599 INFO  (coreLoadExecutor-4385-thread-1) [    ] o.a.s.c.SolrConfig Using Lucene MatchVersion: 7.0.0
   [junit4]   2> 2387604 INFO  (coreLoadExecutor-4385-thread-1) [    ] o.a.s.c.SolrConfig Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2387607 INFO  (coreLoadExecutor-4385-thread-1) [    ] o.a.s.s.IndexSchema [collection1] Schema name=test
   [junit4]   2> 2387609 INFO  (coreLoadExecutor-4385-thread-1) [    ] o.a.s.s.IndexSchema unique key field: id
   [junit4]   2> 2387610 INFO  (coreLoadExecutor-4385-thread-1) [    ] o.a.s.c.ConfigSetProperties Did not find ConfigSet properties, assuming default properties: Can't find resource 'configsetprops.json' in classpath or '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_89FA53BCCCDAFAFA-001/solr-instance-001/collection1'
   [junit4]   2> 2387610 INFO  (coreLoadExecutor-4385-thread-1) [    ] o.a.s.c.CoreContainer Creating SolrCore 'collection1' using configuration from instancedir /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_89FA53BCCCDAFAFA-001/solr-instance-001/./collection1
   [junit4]   2> 2387610 INFO  (coreLoadExecutor-4385-thread-1) [    x:collection1] o.a.s.c.SolrCore org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2387611 INFO  (coreLoadExecutor-4385-thread-1) [    x:collection1] o.a.s.c.SolrCore [[collection1] ] Opening new SolrCore at [/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_89FA53BCCCDAFAFA-001/solr-instance-001/collection1], dataDir=[/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_89FA53BCCCDAFAFA-001/solr-instance-001/./collection1/data/]
   [junit4]   2> 2387611 INFO  (coreLoadExecutor-4385-thread-1) [    x:collection1] o.a.s.c.SolrCore JMX monitoring not detected for core: collection1
   [junit4]   2> 2387611 INFO  (coreLoadExecutor-4385-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_89FA53BCCCDAFAFA-001/solr-instance-001/./collection1/data/snapshot_metadata
   [junit4]   2> 2387611 INFO  (coreLoadExecutor-4385-thread-1) [    x:collection1] o.a.s.c.s.SolrSnapshotMetaDataManager Loading from snapshot metadata file...
   [junit4]   2> 2387611 INFO  (coreLoadExecutor-4385-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_89FA53BCCCDAFAFA-001/solr-instance-001/./collection1/data
   [junit4]   2> 2387611 INFO  (coreLoadExecutor-4385-thread-1) [    x:collection1] o.a.s.c.SolrCore New index directory detected: old=null new=/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_89FA53BCCCDAFAFA-001/solr-instance-001/./collection1/data/index/
   [junit4]   2> 2387611 WARN  (coreLoadExecutor-4385-thread-1) [    x:collection1] o.a.s.c.SolrCore [collection1] Solr index directory '/x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_89FA53BCCCDAFAFA-001/solr-instance-001/./collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 2387612 INFO  (coreLoadExecutor-4385-thread-1) [    x:collection1] o.a.s.c.CachingDirectoryFactory return new directory for /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler_89FA53BCCCDAFAFA-001/solr-instance-001/./collection1/data/index
   [junit4]   2> 2387612 INFO  (coreLoadExecutor-4385-thread-1) [    x:collection1] o.a.s.u.RandomMergePolicy RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@2e512f96
   [junit4]   2> 2387612 INFO  (coreLoadExecutor-4385-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 	commit{dir=MockDirectoryWrapper(RAMDirectory@74a24047 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@715882e0),segFN=segments_1,generation=1}
   [junit4]   2> 2387612 INFO  (coreLoadExecutor-4385-thread-1) [    x:collection1] o.a.s.c.SolrDeletionPolicy newest commit generation = 1
   [junit4]   2> 2387614 INFO  (coreLoadExecutor-4385-thread-1) [    x:collection1] o.a.s.c.SolrCore no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 2387616 INFO  (coreLoadExecutor-4385-thread-1) [    x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 2387616 INFO  (coreLoadExecutor-4385-thread-1) [    x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 2387617 INFO  (coreLoadExecutor-4385-thread-1) [    x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 2387618 INFO  (coreLoadExecutor-4385-thread-1) [    x:collection1] o.a.s.h.l.XMLLoader xsltCacheLifetimeSeconds=60
   [junit4]   2> 2387619 INFO  (coreLoadExecutor-4385-thread-1) [    x:collection1] o.a.s.c.PluginBag requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 2387619 INFO  (coreLoadExecutor-4385-thread-1) [    x:collection1] o.a.s.c.RequestHandlers Registered paths: /admin/mbeans,standard,lazy,/update/json/docs,/admin/luke,/export,/get,/admin/properties,/update/json,/admin/threads,/update/csv,/sql,/graph,/admin/segments,/admin/system,/replication,defaults,/config,/stream,/schema,/admin/plugins,/admin/logging,/adm

[...truncated too long message...]

rClient.java:367)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1280)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:1050)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:992)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1219)
   [junit4]    > 	at org.apache.solr.security.BasicAuthIntegrationTest.doExtraTests(BasicAuthIntegrationTest.java:116)
   [junit4]    > 	at org.apache.solr.cloud.TestMiniSolrCloudClusterBase.testCollectionCreateSearchDelete(TestMiniSolrCloudClusterBase.java:196)
   [junit4]    > 	at org.apache.solr.cloud.TestMiniSolrCloudClusterBase.testBasics(TestMiniSolrCloudClusterBase.java:79)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:34082 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
   [junit4]    > 	at org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
   [junit4]    > 	at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
   [junit4]    > 	at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
   [junit4]    > 	at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
   [junit4]    > 	at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:513)
   [junit4]    > 	... 50 more
   [junit4]   2> NOTE: leaving temporary files on disk at: /x1/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-master/checkout/solr/build/solr-core/test/J0/temp/solr.security.BasicAuthIntegrationTest_89FA53BCCCDAFAFA-001
   [junit4]   2> Aug 30, 2016 10:10:23 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene62): {range_facet_l_dv=PostingsFormat(name=Memory doPackFST= false), _version_=BlockTreeOrds(blocksize=128), multiDefault=PostingsFormat(name=LuceneFixedGap), intDefault=BlockTreeOrds(blocksize=128), id=PostingsFormat(name=Memory doPackFST= false), range_facet_i_dv=BlockTreeOrds(blocksize=128), range_facet_l=BlockTreeOrds(blocksize=128), timestamp=BlockTreeOrds(blocksize=128)}, docValues:{range_facet_l_dv=DocValuesFormat(name=Asserting), range_facet_i_dv=DocValuesFormat(name=Lucene54), timestamp=DocValuesFormat(name=Lucene54)}, maxPointsInLeafNode=685, maxMBSortInHeap=6.664972805357292, sim=ClassicSimilarity, locale=ca, timezone=Atlantic/Jan_Mayen
   [junit4]   2> NOTE: Linux 3.13.0-85-generic amd64/Oracle Corporation 1.8.0_102 (64-bit)/cpus=4,threads=1,free=304068456,total=517472256
   [junit4]   2> NOTE: All tests run in this JVM: [TestRTGBase, CleanupOldIndexTest, DistributedDebugComponentTest, ConnectionReuseTest, RemoteQueryErrorTest, BasicDistributedZkTest, TestLRUCache, StandardRequestHandlerTest, DistributedQueueTest, ActionThrottleTest, TestFieldTypeCollectionResource, TestHighFrequencyDictionaryFactory, RuleEngineTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestRandomCollapseQParserPlugin, SOLR749Test, DeleteReplicaTest, AtomicUpdatesTest, TestSizeLimitedDistributedMap, HttpPartitionTest, TestPKIAuthenticationPlugin, DateRangeFieldTest, DeleteInactiveReplicaTest, TestSQLHandler, TestFieldCacheVsDocValues, TestBlobHandler, TestNumericTerms32, BasicZkTest, TestBinaryField, TestOnReconnectListenerSupport, TestChildDocTransformer, AddBlockUpdateTest, DocExpirationUpdateProcessorFactoryTest, TestDynamicFieldResource, LeaderInitiatedRecoveryOnShardRestartTest, TestGraphTermsQParserPlugin, PreAnalyzedUpdateProcessorTest, DirectoryFactoryTest, TestRecoveryHdfs, TestLMDirichletSimilarityFactory, QueryEqualityTest, WrapperMergePolicyFactoryTest, PingRequestHandlerTest, OverseerModifyCollectionTest, CacheHeaderTest, SmileWriterTest, HLLUtilTest, AnalysisAfterCoreReloadTest, TestShardHandlerFactory, DistributedFacetPivotLongTailTest, TestSolrFieldCacheMBean, TestLazyCores, TestBadConfig, BigEndianAscendingWordDeserializerTest, BufferStoreTest, QueryElevationComponentTest, TestRangeQuery, TestMacroExpander, TestUniqueKeyFieldResource, HdfsThreadLeakTest, CdcrRequestHandlerTest, TestCopyFieldCollectionResource, TestRestManager, TestFaceting, TestSubQueryTransformerDistrib, TestDocSet, SolrCoreTest, JsonLoaderTest, TestSolr4Spatial, CdcrBootstrapTest, TestIBSimilarityFactory, URLClassifyProcessorTest, TestConfigSetImmutable, TestRemoteStreaming, ReturnFieldsTest, TestDFISimilarityFactory, TestOmitPositions, TestRTimerTree, ExplicitHLLTest, TestExpandComponent, HdfsBasicDistributedZkTest, PeerSyncTest, TestNoOpRegenerator, TestHdfsCloudBackupRestore, ZkStateReaderTest, TestConfigSetsAPIExclusivity, ResponseHeaderTest, ResourceLoaderTest, RestartWhileUpdatingTest, SolrCloudExampleTest, TestLegacyNumericRangeQueryBuilder, HdfsRecoveryZkTest, TestSolrCloudWithSecureImpersonation, TestFoldingMultitermQuery, TestRandomDVFaceting, TestJoin, TestRequestStatusCollectionAPI, PolyFieldTest, DistributedFacetPivotWhiteBoxTest, RAMDirectoryFactoryTest, SyncSliceTest, ShardRoutingTest, TestReplicationHandler, AutoCommitTest, TestDistributedGrouping, TestRecovery, TestHashPartitioner, DistributedSpellCheckComponentTest, TermVectorComponentDistributedTest, TestRealTimeGet, TestCoreContainer, SimpleFacetsTest, BadIndexSchemaTest, ConvertedLegacyTest, TestFiltering, TestFunctionQuery, DirectUpdateHandlerTest, TestIndexSearcher, SoftAutoCommitTest, DistributedQueryElevationComponentTest, SuggesterFSTTest, SolrRequestParserTest, TestStressLucene, TestTrie, TestCSVLoader, NoCacheHeaderTest, SchemaVersionSpecificBehaviorTest, TestPseudoReturnFields, TestAtomicUpdateErrorCases, TestWordDelimiterFilterFactory, DefaultValueUpdateProcessorTest, SortByFunctionTest, TestSolrDeletionPolicy1, SolrInfoMBeanTest, XsltUpdateRequestHandlerTest, DebugComponentTest, DisMaxRequestHandlerTest, TestQueryUtils, TestReversedWildcardFilterFactory, FileBasedSpellCheckerTest, RequestHandlersTest, PathHierarchyTokenizerFactoryTest, TermVectorComponentTest, RequiredFieldsTest, TestArbitraryIndexDir, LoggingHandlerTest, RegexBoostProcessorTest, IndexSchemaTest, UpdateRequestProcessorFactoryTest, MBeansHandlerTest, TestAnalyzedSuggestions, TestPHPSerializedResponseWriter, TestComponentsName, TestQuerySenderListener, UpdateParamsTest, AlternateDirectoryTest, CopyFieldTest, ResponseLogComponentTest, SolrIndexConfigTest, BadComponentTest, TestMergePolicyConfig, MultiTermTest, SampleTest, TestSearchPerf, NumericFieldsTest, MinimalSchemaTest, TestConfig, OutputWriterTest, ExternalFileFieldSortTest, TestPostingsSolrHighlighter, DirectSolrConnectionTest, TestLuceneMatchVersion, TestCharFilters, SynonymTokenizerTest, TestXIncludeConfig, EchoParamsTest, TestPerFieldSimilarity, TestFastWriter, DateMathParserTest, TestSystemIdResolver, PrimUtilsTest, DateFieldTest, ZkNodePropsTest, SystemInfoHandlerTest, FileUtilsTest, CursorPagingTest, SolrTestCaseJ4Test, TestCrossCoreJoin, TestCursorMarkWithoutUniqueKey, TestHighlightDedupGrouping, TestEmbeddedSolrServerConstructors, TestJettySolrRunner, AsyncCallRequestStatusResponseTest, CdcrReplicationDistributedZkTest, HdfsWriteToMultipleCollectionsTest, BlobRepositoryCloudTest, CoreSorterTest, QueryResultKeyTest, TestConfigOverlay, TestConfigSets, TestDynamicLoading, TestInitParams, TestNRTOpen, TestReloadAndDeleteDocs, TestSolrConfigHandler, TestSolrXml, TestSolrCoreSnapshots, RequestLoggingTest, TestHdfsBackupRestoreCore, TestRestoreCore, CoreAdminCreateDiscoverTest, CoreAdminRequestStatusTest, InfoHandlerTest, SegmentsInfoRequestHandlerTest, DistributedExpandComponentTest, DistributedFacetPivotSmallAdvancedTest, DistributedQueryComponentCustomSortTest, DistributedQueryComponentOptimizationTest, SuggestComponentContextFilterQueryTest, SuggestComponentTest, TestTrackingShardHandlerFactory, TestIntervalFaceting, TestGeoJSONResponseWriter, TestGraphMLResponseWriter, TestRawTransformer, TestSolrQueryResponse, TestManagedResourceStorage, TestDynamicFieldCollectionResource, TestSchemaResource, TestSchemaSimilarityResource, TestSchemaVersionResource, TestSerializedLuceneMatchVersion, TestSolrQueryParserDefaultOperatorResource, TestManagedSynonymFilterFactory, BadCopyFieldTest, BooleanFieldTest, ChangedSchemaMergeTest, DocValuesMissingTest, SpatialRPTFieldTypeTest, TestCloudManagedSchema, TestCloudSchemaless, TestReRankQParserPlugin, TestSimpleQParserPlugin, TestSolr4Spatial2, TestJsonFacets, TestJsonRequest, CloudMLTQParserTest, SimpleMLTQParserTest, TestNonDefinedSimilarityFactory, TestPerFieldSimilarityWithDefaultOverride, TestDistribIDF, BasicAuthIntegrationTest]
   [junit4] Completed [606/632 (6!)] on J0 in 21.73s, 1 test, 1 error <<< FAILURES!

[...truncated 60176 lines...]