You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2015/02/25 17:27:39 UTC

[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 769 - Still Failing

Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/769/

5 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([EAC8C0E6A3EA6233:629CFF3C0D160FCB]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:887)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:752)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:568)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:565)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([EAC8C0E6A3EA6233:629CFF3C0D160FCB]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:887)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:752)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:568)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:565)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:47768/rsa/gf/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:47768/rsa/gf/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([EAC8C0E6A3EA6233:629CFF3C0D160FCB]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:131)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:176)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:154)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 49 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([EAC8C0E6A3EA6233:629CFF3C0D160FCB]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:887)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:752)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:131)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:110)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:568)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:565)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([EAC8C0E6A3EA6233:629CFF3C0D160FCB]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:887)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:752)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:568)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:565)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 10130 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/init-core-data-001
   [junit4]   2> 93131 T300 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /rsa/gf
   [junit4]   2> 93142 T300 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 93143 T301 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 93243 T300 oasc.ZkTestServer.run start zk server on port:47640
   [junit4]   2> 93244 T300 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 93245 T300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 93249 T308 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@35f43ae0 name:ZooKeeperConnection Watcher:127.0.0.1:47640 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 93249 T300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 93250 T300 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 93250 T300 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 93254 T300 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 93254 T300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 93256 T311 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@183b70a2 name:ZooKeeperConnection Watcher:127.0.0.1:47640/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 93256 T300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 93257 T300 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 93266 T300 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 93270 T300 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 93272 T300 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 93275 T300 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 93278 T300 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 93279 T300 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 93282 T300 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 93283 T300 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 93286 T300 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 93287 T300 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 93289 T300 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 93290 T300 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 93292 T300 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 93293 T300 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 93296 T300 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 93296 T300 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 93299 T300 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 93299 T300 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 93302 T300 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 93303 T300 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 93305 T300 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 93306 T300 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 93308 T300 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 93309 T300 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 93312 T300 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 93312 T300 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 94824 T300 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/control-001/cores/collection1
   [junit4]   2> 94829 T300 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 94833 T300 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:47656
   [junit4]   2> 94834 T300 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3972aa3f
   [junit4]   2> 94834 T300 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 94834 T300 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/control-001
   [junit4]   2> 94835 T300 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/control-001/'
   [junit4]   2> 94880 T300 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/control-001/solr.xml
   [junit4]   2> 94941 T300 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/control-001/cores
   [junit4]   2> 94942 T300 oasc.CoreContainer.<init> New CoreContainer 593775859
   [junit4]   2> 94942 T300 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/control-001/]
   [junit4]   2> 94943 T300 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 94944 T300 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 94944 T300 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 94944 T300 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 94944 T300 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 94944 T300 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 94945 T300 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 94945 T300 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 94945 T300 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 94945 T300 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 94946 T300 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 94946 T300 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 94947 T300 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 94947 T300 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 94947 T300 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 94947 T300 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:47640/solr
   [junit4]   2> 95001 T300 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 95001 T300 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 95002 T300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 95005 T323 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3dc342a9 name:ZooKeeperConnection Watcher:127.0.0.1:47640 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 95005 T300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 95006 T300 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 95036 T300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 95038 T326 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@78b5282b name:ZooKeeperConnection Watcher:127.0.0.1:47640/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 95038 T300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 95063 T300 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 95078 T300 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 95089 T300 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 95093 T300 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 95098 T300 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 95130 T300 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 95133 T300 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:47655_rsa%2Fgf
   [junit4]   2> 95134 T300 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:47655_rsa%2Fgf
   [junit4]   2> 95140 T300 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 95143 T300 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 95146 T300 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 95157 T300 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:47655_rsa%2Fgf
   [junit4]   2> 95158 T300 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 95162 T300 oasc.Overseer.start Overseer (id=93380773926141955-127.0.0.1:47655_rsa%2Fgf-n_0000000000) starting
   [junit4]   2> 95210 T300 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 95251 T300 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 95283 T328 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 95287 T300 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 95290 T300 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 95293 T300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 95309 T327 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 95326 T300 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/control-001/cores
   [junit4]   2> 95350 T300 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/control-001/cores/collection1/
   [junit4]   2> 95350 T300 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 95352 T330 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 95352 T330 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 95359 T326 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 95359 T330 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 95370 T327 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:47655_rsa%2Fgf",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:47655/rsa/gf",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 95440 T327 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:47655_rsa%2Fgf",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:47655/rsa/gf",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 95450 T327 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 95513 T327 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 95517 T326 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 96361 T330 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 96361 T330 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 96362 T330 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 96362 T330 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 96363 T330 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 96364 T330 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/control-001/cores/collection1/'
   [junit4]   2> 96411 T330 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 96435 T330 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 96471 T330 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 96504 T330 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 96561 T330 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 96563 T330 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 96584 T330 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 97062 T330 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 97076 T330 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 97080 T330 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 97107 T330 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 97113 T330 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 97119 T330 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 97121 T330 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 97121 T330 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 97122 T330 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 97123 T330 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 97123 T330 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 97124 T330 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 97125 T330 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 97125 T330 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/tempDir-001/control/data/
   [junit4]   2> 97126 T330 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3900094c
   [junit4]   2> 97127 T330 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/tempDir-001/control/data
   [junit4]   2> 97127 T330 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/tempDir-001/control/data/index/
   [junit4]   2> 97127 T330 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 97128 T330 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/tempDir-001/control/data/index
   [junit4]   2> 97128 T330 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=47, maxMergeAtOnceExplicit=34, maxMergedSegmentMB=16.0556640625, floorSegmentMB=0.8310546875, forceMergeDeletesPctAllowed=26.90701564590275, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5980113102276826
   [junit4]   2> 97129 T330 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6f99af5a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@52c776f3),segFN=segments_1,generation=1}
   [junit4]   2> 97129 T330 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 97139 T330 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 97140 T330 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 97140 T330 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 97140 T330 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 97141 T330 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 97142 T330 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 97142 T330 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 97142 T330 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 97143 T330 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 97144 T330 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 97145 T330 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 97145 T330 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 97146 T330 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 97147 T330 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 97148 T330 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 97149 T330 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 97149 T330 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 97150 T330 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 97152 T330 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 97153 T330 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 97153 T330 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 97154 T330 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 97155 T330 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 97156 T330 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 97157 T330 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 97158 T330 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 97158 T330 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 97178 T330 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 97181 T330 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 97183 T330 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 97185 T330 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 97188 T330 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 97192 T330 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 97192 T330 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 97194 T330 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=33, maxMergeAtOnceExplicit=30, maxMergedSegmentMB=72.2490234375, floorSegmentMB=1.16796875, forceMergeDeletesPctAllowed=10.971804425252458, segmentsPerTier=16.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 97195 T330 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6f99af5a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@52c776f3),segFN=segments_1,generation=1}
   [junit4]   2> 97195 T330 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 97195 T330 oass.SolrIndexSearcher.<init> Opening Searcher@705151f[collection1] main
   [junit4]   2> 97196 T330 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 97198 T330 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 97198 T330 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 97208 T330 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 97208 T330 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 97208 T330 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 97209 T330 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 97209 T330 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 97210 T330 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 97210 T330 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 97211 T331 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@705151f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 97215 T330 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 97216 T334 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:47655/rsa/gf collection:control_collection shard:shard1
   [junit4]   2> 97217 T300 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 97217 T300 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 97238 T334 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 97245 T334 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 97247 T326 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 97248 T334 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 97248 T334 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C72 name=collection1 org.apache.solr.core.SolrCore@71c37e41 url=http://127.0.0.1:47655/rsa/gf/collection1 node=127.0.0.1:47655_rsa%2Fgf C72_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, node_name=127.0.0.1:47655_rsa%2Fgf, base_url=http://127.0.0.1:47655/rsa/gf}
   [junit4]   2> 97248 T334 C72 P47655 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:47655/rsa/gf/collection1/
   [junit4]   2> 97249 T327 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 97249 T334 C72 P47655 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 97249 T334 C72 P47655 oasc.SyncStrategy.syncToMe http://127.0.0.1:47655/rsa/gf/collection1/ has no replicas
   [junit4]   2> 97249 T334 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:47655/rsa/gf/collection1/ shard1
   [junit4]   2> 97253 T300 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 97254 T300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 97256 T338 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@744c2cdd name:ZooKeeperConnection Watcher:127.0.0.1:47640/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 97257 T300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 97257 T300 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 97259 T326 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 97260 T300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 97260 T334 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 97267 T300 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 97268 T326 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 97269 T327 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:47655/rsa/gf",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 97375 T326 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 97375 T338 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 97422 T334 oasc.ZkController.register We are http://127.0.0.1:47655/rsa/gf/collection1/ and leader is http://127.0.0.1:47655/rsa/gf/collection1/
   [junit4]   2> 97422 T334 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:47655/rsa/gf
   [junit4]   2> 97422 T334 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 97422 T334 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 97423 T334 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 97425 T326 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 97426 T327 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:47655_rsa%2Fgf",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:47655/rsa/gf",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1"} current state version: 3
   [junit4]   2> 97427 T327 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:47655_rsa%2Fgf",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:47655/rsa/gf",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 97533 T338 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 97533 T326 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 98421 T300 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/shard-1-001/cores/collection1
   [junit4]   2> 98424 T300 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 98425 T300 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 98427 T300 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:47691
   [junit4]   2> 98427 T300 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3972aa3f
   [junit4]   2> 98428 T300 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 98428 T300 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/shard-1-001
   [junit4]   2> 98428 T300 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/shard-1-001/'
   [junit4]   2> 98466 T300 oasc.ConfigSolr.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/shard-1-001/solr.xml
   [junit4]   2> 98529 T300 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/shard-1-001/cores
   [junit4]   2> 98530 T300 oasc.CoreContainer.<init> New CoreContainer 589087578
   [junit4]   2> 98530 T300 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/shard-1-001/]
   [junit4]   2> 98534 T300 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 98535 T300 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 98535 T300 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 98535 T300 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 98535 T300 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 98536 T300 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 98536 T300 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 98536 T300 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 98536 T300 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 98537 T300 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 98537 T300 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 98537 T300 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 98538 T300 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 98538 T300 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 98538 T300 oasc.CoreContainer.load Host Name: 127.0.0.1
   [junit4]   2> 98538 T300 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:47640/solr
   [junit4]   2> 98539 T300 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 98539 T300 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 98540 T300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 98542 T350 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7e44a305 name:ZooKeeperConnection Watcher:127.0.0.1:47640 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 98542 T300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 98543 T300 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 98546 T300 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 98548 T353 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1e38914f name:ZooKeeperConnection Watcher:127.0.0.1:47640/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 98548 T300 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 98557 T300 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 99563 T300 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:47690_rsa%2Fgf
   [junit4]   2> 99565 T300 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:47690_rsa%2Fgf
   [junit4]   2> 99569 T300 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 99580 T300 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/shard-1-001/cores
   [junit4]   2> 99583 T300 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/shard-1-001/cores/collection1/
   [junit4]   2> 99583 T300 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 99585 T354 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 99585 T354 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 99587 T354 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 99587 T326 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 99589 T327 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:47690_rsa%2Fgf",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:47690/rsa/gf",
   [junit4]   2> 	  "shard":null} current state version: 4
   [junit4]   2> 99590 T327 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:47690_rsa%2Fgf",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:47690/rsa/gf",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 99590 T327 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 99590 T327 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 99593 T353 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 99593 T338 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 99593 T326 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 100587 T354 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 100588 T354 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 100589 T354 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 100589 T354 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 100591 T354 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 100591 T354 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/shard-1-001/cores/collection1/'
   [junit4]   2> 100610 T354 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 100614 T354 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 100629 T354 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 100655 T354 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 100713 T354 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 100715 T354 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 100734 T354 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 101081 T354 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 101090 T354 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 101093 T354 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 101115 T354 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 101120 T354 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 101125 T354 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 101126 T354 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 101127 T354 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 101127 T354 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 101128 T354 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 101129 T354 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 101129 T354 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 101129 T354 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 101129 T354 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/tempDir-001/jetty1/
   [junit4]   2> 101130 T354 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3900094c
   [junit4]   2> 101130 T354 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/tempDir-001/jetty1
   [junit4]   2> 101131 T354 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/tempDir-001/jetty1/index/
   [junit4]   2> 101131 T354 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/tempDir-001/jetty1/index' doesn't exist. Creating new index...
   [junit4]   2> 101131 T354 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest EAC8C0E6A3EA6233-001/tempDir-001/jetty1/index
   [junit4]   2> 101131 T354 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=47, maxMergeAtOnceExplicit=34, maxMergedSegmentMB=16.0556640625, floorSegmentMB=0.8310546875, forceMergeDeletesPctAllowed=26.90701564590275, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5980113102276826
   [junit4]   2> 101132 T354 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7620d6ee lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@65fedfd8),segFN=segments_1,generation=1}
   [junit4]   2> 101132 T354 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 101141 T354 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 101141 T354 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 101141 T354 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 101141 T354 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 101142 T354 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 101142 T354 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 101142 T354 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 101142 T354 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 101143 T354 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 101144 T354 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 101144 T354 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 101145 T354 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 101146 T354 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 101147 T354 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 101147 T354 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 101149 T354 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 101149 T354 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 101150 T354 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 101152 T354 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 101153 T354 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 101153 T354 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 101154 T354 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 101155 T354 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 101156 T354 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 101156 T35

[...truncated too long message...]

losing 5 connections to: http://127.0.0.1:44340/jegx/hs
   [junit4]   2> 2312616 T9067 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:44315/jegx/hs
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=EAC8C0E6A3EA6233 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=fr -Dtests.timezone=America/Indiana/Tell_City -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   64.1s J0 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([EAC8C0E6A3EA6233:629CFF3C0D160FCB]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:887)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:752)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:131)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:124)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:110)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:940)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:915)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:568)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:565)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 2312629 T9067 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryAfterSoftCommitTest EAC8C0E6A3EA6233-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {a_t=PostingsFormat(name=LuceneFixedGap), text=PostingsFormat(name=Memory doPackFST= true), intDefault=PostingsFormat(name=MockRandom), _version_=PostingsFormat(name=MockRandom), range_facet_l=PostingsFormat(name=LuceneFixedGap), timestamp=PostingsFormat(name=LuceneFixedGap), id=PostingsFormat(name=Memory doPackFST= false), multiDefault=PostingsFormat(name=LuceneFixedGap)}, docValues:{timestamp=DocValuesFormat(name=Direct)}, sim=DefaultSimilarity, locale=fr, timezone=America/Indiana/Tell_City
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=166693640,total=384827392
   [junit4]   2> NOTE: All tests run in this JVM: [DebugComponentTest, SearchHandlerTest, TestStressReorder, ActionThrottleTest, ZkSolrClientTest, TestNoOpRegenerator, ShowFileRequestHandlerTest, HighlighterMaxOffsetTest, EnumFieldTest, DOMUtilTest, LeaderInitiatedRecoveryOnCommitTest, HdfsLockFactoryTest, TestRecoveryHdfs, TestCollationField, HighlighterConfigTest, DocValuesMissingTest, TestBulkSchemaConcurrent, HdfsRecoveryZkTest, TestCharFilters, TestSchemaVersionResource, TestMissingGroups, TestIBSimilarityFactory, DistanceFunctionTest, TestInitParams, TestFreeTextSuggestions, SimpleFacetsTest, RequestHandlersTest, QueryParsingTest, RollingRestartTest, TestRequestStatusCollectionAPI, TestReRankQParserPlugin, OverseerRolesTest, MBeansHandlerTest, TestManagedResourceStorage, AnalysisErrorHandlingTest, TestSolrConfigHandlerCloud, ReplicationFactorTest, UUIDFieldTest, TestRandomFaceting, BasicFunctionalityTest, TestFieldResource, ParsingFieldUpdateProcessorsTest, TestMiniSolrCloudClusterSSL, TestSolrDeletionPolicy2, HdfsBasicDistributedZkTest, OpenExchangeRatesOrgProviderTest, BasicDistributedZkTest, TestSort, PolyFieldTest, DirectSolrSpellCheckerTest, TestDefaultStatsCache, SuggesterTest, TestCoreContainer, BlockCacheTest, TestFaceting, TestUpdate, ExternalFileFieldSortTest, LeaderElectionTest, TestRandomDVFaceting, ShardRoutingCustomTest, AutoCommitTest, TestHashPartitioner, DistributedSpellCheckComponentTest, TermVectorComponentDistributedTest, TestReload, DistributedTermsComponentTest, TestRangeQuery, SolrCoreTest, TestGroupingSearch, PeerSyncTest, ConvertedLegacyTest, DirectUpdateHandlerTest, TestBadConfig, DistributedQueryElevationComponentTest, SimplePostToolTest, AnalysisAfterCoreReloadTest, SpellCheckCollatorTest, SolrRequestParserTest, TestStressLucene, SuggesterWFSTTest, SolrCoreCheckLockOnStartupTest, TestAtomicUpdateErrorCases, StatelessScriptUpdateProcessorFactoryTest, DocValuesMultiTest, SolrInfoMBeanTest, TestQueryUtils, DocumentAnalysisRequestHandlerTest, FileBasedSpellCheckerTest, XmlUpdateRequestHandlerTest, TestValueSourceCache, PathHierarchyTokenizerFactoryTest, RequiredFieldsTest, FastVectorHighlighterTest, RegexBoostProcessorTest, IndexSchemaTest, JsonLoaderTest, TestComponentsName, TestQuerySenderListener, TestQuerySenderNoQuery, BadComponentTest, TestStressRecovery, CursorPagingTest, ConnectionManagerTest, CustomCollectionTest, DeleteShardTest, DistribJoinFromCollectionTest, DistributedQueueTest, ExternalCollectionsTest, OverseerStatusTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J0 in 64.09s, 1 test, 1 error <<< FAILURES!

[...truncated 272 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:508: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1368: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:975: There were test failures: 467 suites, 1863 tests, 5 errors, 56 ignored (9 assumptions)

Total time: 140 minutes 11 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 719100783 bytes
Compression is 0.0%
Took 2 min 9 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 816 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/816/

4 tests failed.
REGRESSION:  org.apache.solr.cloud.CollectionsAPIDistributedZkTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=4821, name=collection3, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=4821, name=collection3, state=RUNNABLE, group=TGRP-CollectionsAPIDistributedZkTest]
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:27346: Could not find collection : awholynewstresscollection_collection3_1
	at __randomizedtesting.SeedInfo.seed([336CA9E425036561]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:557)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:234)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:226)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:1068)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:840)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:783)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.CollectionsAPIDistributedZkTest$1CollectionThread.run(CollectionsAPIDistributedZkTest.java:884)


REGRESSION:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
expected:<1> but was:<3>

Stack Trace:
java.lang.AssertionError: expected:<1> but was:<3>
	at __randomizedtesting.SeedInfo.seed([336CA9E425036561:BB38963E8BFF0899]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testDeleteByIdCompositeRouterWithRouterField(FullSolrCloudDistribCmdsTest.java:397)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test(FullSolrCloudDistribCmdsTest.java:146)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.request.TestWriterPerf.testPerf

Error Message:


Stack Trace:
java.lang.NullPointerException
	at __randomizedtesting.SeedInfo.seed([336CA9E425036561:125B48634D8D4E50]:0)
	at org.apache.solr.highlight.DefaultSolrHighlighter.doHighlightingByHighlighter(DefaultSolrHighlighter.java:452)
	at org.apache.solr.highlight.DefaultSolrHighlighter.doHighlighting(DefaultSolrHighlighter.java:395)
	at org.apache.solr.handler.component.HighlightComponent.process(HighlightComponent.java:143)
	at org.apache.solr.handler.component.SearchHandler.handleRequestBody(SearchHandler.java:253)
	at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:143)
	at org.apache.solr.core.SolrCore.execute(SolrCore.java:1988)
	at org.apache.solr.request.TestWriterPerf.getResponse(TestWriterPerf.java:96)
	at org.apache.solr.request.TestWriterPerf.doPerf(TestWriterPerf.java:105)
	at org.apache.solr.request.TestWriterPerf.testPerf(TestWriterPerf.java:178)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.request.TestWriterPerf

Error Message:
1

Stack Trace:
java.lang.AssertionError: 1
	at __randomizedtesting.SeedInfo.seed([336CA9E425036561]:0)
	at org.apache.solr.core.CachingDirectoryFactory.close(CachingDirectoryFactory.java:201)
	at org.apache.solr.core.SolrCore.close(SolrCore.java:1176)
	at org.apache.solr.core.SolrCores.close(SolrCores.java:117)
	at org.apache.solr.core.CoreContainer.shutdown(CoreContainer.java:378)
	at org.apache.solr.util.TestHarness.close(TestHarness.java:359)
	at org.apache.solr.SolrTestCaseJ4.deleteCore(SolrTestCaseJ4.java:704)
	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:230)
	at sun.reflect.GeneratedMethodAccessor32.invoke(Unknown Source)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:799)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 10695 lines...]
   [junit4] Suite: org.apache.solr.cloud.CollectionsAPIDistributedZkTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/init-core-data-001
   [junit4]   2> 1254319 T4324 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (true)
   [junit4]   2> 1254319 T4324 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1254325 T4324 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   2> 1254325 T4325 oasc.ZkTestServer$2$1.setClientPort client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1254326 T4325 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1254426 T4324 oasc.ZkTestServer.run start zk server on port:63380
   [junit4]   2> 1254427 T4324 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1254433 T4324 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1254440 T4332 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@780cc69a name:ZooKeeperConnection Watcher:127.0.0.1:63380 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1254441 T4324 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1254441 T4324 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1254442 T4324 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1254445 T4324 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1254447 T4324 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1254448 T4335 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@f806a6c name:ZooKeeperConnection Watcher:127.0.0.1:63380/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1254448 T4324 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1254448 T4324 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1254449 T4324 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1254450 T4324 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1254451 T4324 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1254453 T4324 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1254454 T4324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1254454 T4324 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1254456 T4324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1254457 T4324 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1254559 T4324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1254560 T4324 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1254562 T4324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1254562 T4324 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1254564 T4324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1254565 T4324 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1254566 T4324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1254567 T4324 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1254569 T4324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1254569 T4324 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1254572 T4324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1254572 T4324 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1254574 T4324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1254574 T4324 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1254577 T4324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1254577 T4324 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1254579 T4324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1254580 T4324 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1254582 T4324 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1254582 T4324 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1254584 T4338 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@208189a6 name:ZooKeeperConnection Watcher:127.0.0.1:63380 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1254584 T4324 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1254585 T4324 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1254585 T4324 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1254587 T4324 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1254587 T4324 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1254588 T4341 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3ad5db99 name:ZooKeeperConnection Watcher:127.0.0.1:63380/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1254589 T4324 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1254589 T4324 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1254591 T4324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf2/solrconfig.xml
   [junit4]   2> 1254592 T4324 oascc.SolrZkClient.makePath makePath: /configs/conf2/solrconfig.xml
   [junit4]   2> 1254594 T4324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf2/schema.xml
   [junit4]   2> 1254595 T4324 oascc.SolrZkClient.makePath makePath: /configs/conf2/schema.xml
   [junit4]   2> 1254696 T4324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf2/enumsConfig.xml
   [junit4]   2> 1254697 T4324 oascc.SolrZkClient.makePath makePath: /configs/conf2/enumsConfig.xml
   [junit4]   2> 1254699 T4324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf2/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1254699 T4324 oascc.SolrZkClient.makePath makePath: /configs/conf2/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1254700 T4324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf2/stopwords.txt
   [junit4]   2> 1254701 T4324 oascc.SolrZkClient.makePath makePath: /configs/conf2/stopwords.txt
   [junit4]   2> 1254702 T4324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf2/protwords.txt
   [junit4]   2> 1254703 T4324 oascc.SolrZkClient.makePath makePath: /configs/conf2/protwords.txt
   [junit4]   2> 1254704 T4324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf2/currency.xml
   [junit4]   2> 1254705 T4324 oascc.SolrZkClient.makePath makePath: /configs/conf2/currency.xml
   [junit4]   2> 1254706 T4324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf2/open-exchange-rates.json
   [junit4]   2> 1254707 T4324 oascc.SolrZkClient.makePath makePath: /configs/conf2/open-exchange-rates.json
   [junit4]   2> 1254708 T4324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf2/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1254708 T4324 oascc.SolrZkClient.makePath makePath: /configs/conf2/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1254710 T4324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf2/old_synonyms.txt
   [junit4]   2> 1254710 T4324 oascc.SolrZkClient.makePath makePath: /configs/conf2/old_synonyms.txt
   [junit4]   2> 1254711 T4324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf2/synonyms.txt
   [junit4]   2> 1254712 T4324 oascc.SolrZkClient.makePath makePath: /configs/conf2/synonyms.txt
   [junit4]   2> 1254713 T4324 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/elevate.xml to /configs/conf2/elevate.xml
   [junit4]   2> 1254714 T4324 oascc.SolrZkClient.makePath makePath: /configs/conf2/elevate.xml
   [junit4]   2> 1255700 T4324 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/control-001/cores/collection1
   [junit4]   2> 1255703 T4324 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1255707 T4324 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:63147
   [junit4]   2> 1255708 T4324 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/control-001/cores, hostContext=/, hostPort=63147}
   [junit4]   2> 1255708 T4324 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 1255708 T4324 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/control-001/'
   [junit4]   2> 1255738 T4324 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/control-001/solr.xml
   [junit4]   2> 1255773 T4324 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/control-001/cores
   [junit4]   2> 1255774 T4324 oasc.CoreContainer.<init> New CoreContainer 497224668
   [junit4]   2> 1255774 T4324 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/control-001/]
   [junit4]   2> 1255774 T4324 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/control-001/lib
   [junit4]   2> 1255775 T4324 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/control-001/lib).
   [junit4]   2> 1255784 T4324 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1255784 T4324 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1255784 T4324 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1255784 T4324 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1255785 T4324 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1255785 T4324 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1255785 T4324 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1255785 T4324 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1255785 T4324 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1255786 T4324 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1255786 T4324 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1255790 T4324 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1255791 T4324 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1255791 T4324 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1255791 T4324 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1255792 T4324 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:63380/solr
   [junit4]   2> 1255792 T4324 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1255792 T4324 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1255793 T4324 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1255795 T4353 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3a03126b name:ZooKeeperConnection Watcher:127.0.0.1:63380 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1255796 T4324 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1255797 T4324 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1255799 T4324 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1255800 T4356 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4a9d3a48 name:ZooKeeperConnection Watcher:127.0.0.1:63380/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1255800 T4324 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1255802 T4324 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1255804 T4324 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1255806 T4324 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1255808 T4324 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1255810 T4324 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1255811 T4324 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1255813 T4324 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1255814 T4324 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1255815 T4324 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63147_
   [junit4]   2> 1255816 T4324 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:63147_
   [junit4]   2> 1255817 T4324 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1255819 T4324 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1255820 T4324 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1255822 T4324 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:63147_
   [junit4]   2> 1255822 T4324 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1255823 T4324 oasc.Overseer.start Overseer (id=93647043439362053-127.0.0.1:63147_-n_0000000000) starting
   [junit4]   2> 1255825 T4324 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1255831 T4324 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=10000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1255832 T4358 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1255832 T4324 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1255832 T4357 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1255836 T4324 N:127.0.0.1:63147_ oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/control-001/cores
   [junit4]   2> 1255838 T4324 N:127.0.0.1:63147_ oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=control_collection, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/control-001/cores/collection1/, dataDir=data/, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/control-001/cores/collection1, shard=, config=solrconfig.xml, schema=schema.xml, name=collection1, coreNodeName=, transient=false}
   [junit4]   2> 1255839 T4324 N:127.0.0.1:63147_ oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/control-001/cores/collection1/
   [junit4]   2> 1255839 T4324 N:127.0.0.1:63147_ oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1255840 T4360 N:127.0.0.1:63147_ C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1255840 T4360 N:127.0.0.1:63147_ C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1255841 T4356 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1255841 T4360 N:127.0.0.1:63147_ oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1255842 T4357 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63147",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:63147_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down"} current state version: 0
   [junit4]   2> 1255842 T4357 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63147",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:63147_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 1255842 T4357 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1255843 T4357 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1255844 T4356 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1256841 T4360 N:127.0.0.1:63147_ oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1256842 T4360 N:127.0.0.1:63147_ oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1256842 T4360 N:127.0.0.1:63147_ oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1256842 T4360 N:127.0.0.1:63147_ oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1256843 T4360 N:127.0.0.1:63147_ oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1256843 T4360 N:127.0.0.1:63147_ oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/control-001/cores/collection1/'
   [junit4]   2> 1256856 T4360 N:127.0.0.1:63147_ oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1256858 T4360 N:127.0.0.1:63147_ oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1256866 T4360 N:127.0.0.1:63147_ oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1256881 T4360 N:127.0.0.1:63147_ oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1256915 T4360 N:127.0.0.1:63147_ oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1257016 T4360 N:127.0.0.1:63147_ oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1257023 T4360 N:127.0.0.1:63147_ oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1257302 T4360 N:127.0.0.1:63147_ oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1257312 T4360 N:127.0.0.1:63147_ oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1257314 T4360 N:127.0.0.1:63147_ oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1257323 T4360 N:127.0.0.1:63147_ oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1257327 T4360 N:127.0.0.1:63147_ oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1257330 T4360 N:127.0.0.1:63147_ oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1257331 T4360 N:127.0.0.1:63147_ oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1257331 T4360 N:127.0.0.1:63147_ oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1257331 T4360 N:127.0.0.1:63147_ oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1257332 T4360 N:127.0.0.1:63147_ oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1257332 T4360 N:127.0.0.1:63147_ oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1257332 T4360 N:127.0.0.1:63147_ oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1257333 T4360 N:127.0.0.1:63147_ c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1257333 T4360 N:127.0.0.1:63147_ c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/control-001/cores/collection1/data/
   [junit4]   2> 1257333 T4360 N:127.0.0.1:63147_ c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2127155
   [junit4]   2> 1257334 T4360 N:127.0.0.1:63147_ c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/control-001/cores/collection1/data
   [junit4]   2> 1257334 T4360 N:127.0.0.1:63147_ c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1257334 T4360 N:127.0.0.1:63147_ c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1257335 T4360 N:127.0.0.1:63147_ c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/control-001/cores/collection1/data/index
   [junit4]   2> 1257335 T4360 N:127.0.0.1:63147_ c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=34, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8097364795430507]
   [junit4]   2> 1257336 T4360 N:127.0.0.1:63147_ c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@7b08645a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@63f48fd0; maxCacheMB=0.12109375 maxMergeSizeMB=0.826171875)),segFN=segments_1,generation=1}
   [junit4]   2> 1257336 T4360 N:127.0.0.1:63147_ c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1257340 T4360 N:127.0.0.1:63147_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1257340 T4360 N:127.0.0.1:63147_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1257341 T4360 N:127.0.0.1:63147_ c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1257341 T4360 N:127.0.0.1:63147_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1257341 T4360 N:127.0.0.1:63147_ c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1257341 T4360 N:127.0.0.1:63147_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1257342 T4360 N:127.0.0.1:63147_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1257342 T4360 N:127.0.0.1:63147_ c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1257342 T4360 N:127.0.0.1:63147_ c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1257357 T4360 N:127.0.0.1:63147_ c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1257359 T4360 N:127.0.0.1:63147_ c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1257360 T4360 N:127.0.0.1:63147_ c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1257362 T4360 N:127.0.0.1:63147_ c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1257365 T4360 N:127.0.0.1:63147_ c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/plugins,/update/csv,/admin/mbeans,/admin/properties,/schema,/update/json,/update/json/docs,/admin/threads,/admin/luke,/admin/file,/admin/segments,standard,/admin/system,/update,/admin/logging,/replication,/config,/admin/ping,/get
   [junit4]   2> 1257367 T4360 N:127.0.0.1:63147_ c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1257368 T4360 N:127.0.0.1:63147_ c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1257368 T4360 N:127.0.0.1:63147_ c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1257369 T4360 N:127.0.0.1:63147_ c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1257370 T4360 N:127.0.0.1:63147_ c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1257370 T4360 N:127.0.0.1:63147_ c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=37, maxMergeAtOnceExplicit=11, maxMergedSegmentMB=51.904296875, floorSegmentMB=0.9609375, forceMergeDeletesPctAllowed=11.590078393234766, segmentsPerTier=41.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6104978885513848
   [junit4]   2> 1257371 T4360 N:127.0.0.1:63147_ c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@7b08645a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@63f48fd0; maxCacheMB=0.12109375 maxMergeSizeMB=0.826171875)),segFN=segments_1,generation=1}
   [junit4]   2> 1257371 T4360 N:127.0.0.1:63147_ c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1257371 T4360 N:127.0.0.1:63147_ c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4583e198[collection1] main
   [junit4]   2> 1257371 T4360 N:127.0.0.1:63147_ c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1257372 T4360 N:127.0.0.1:63147_ c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1257372 T4360 N:127.0.0.1:63147_ c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1257373 T4360 N:127.0.0.1:63147_ c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1257373 T4360 N:127.0.0.1:63147_ c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1257373 T4360 N:127.0.0.1:63147_ c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1257374 T4360 N:127.0.0.1:63147_ c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1257374 T4360 N:127.0.0.1:63147_ c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1257374 T4360 N:127.0.0.1:63147_ c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1257374 T4360 N:127.0.0.1:63147_ c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1257375 T4360 N:127.0.0.1:63147_ c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1257375 T4361 N:127.0.0.1:63147_ c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4583e198[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1257375 T4364 N:127.0.0.1:63147_ C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:63147 collection:control_collection shard:shard1
   [junit4]   2> 1257376 T4324 N:127.0.0.1:63147_ oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 1257376 T4324 N:127.0.0.1:63147_ oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1257376 T4364 N:127.0.0.1:63147_ C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1257379 T4324 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1257380 T4324 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1257381 T4367 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2c13f269 name:ZooKeeperConnection Watcher:127.0.0.1:63380/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1257382 T4364 N:127.0.0.1:63147_ C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1257382 T4324 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1257382 T4324 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1257383 T4324 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1257383 T4356 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1257383 T4364 N:127.0.0.1:63147_ C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1257383 T4364 N:127.0.0.1:63147_ C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C562 name=collection1 org.apache.solr.core.SolrCore@5a946575 url=http://127.0.0.1:63147/collection1 node=127.0.0.1:63147_ C562_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=http://127.0.0.1:63147, node_name=127.0.0.1:63147_, state=down}
   [junit4]   2> 1257384 T4364 N:127.0.0.1:63147_ C:control_collection S:shard1 C562 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:63147/collection1/
   [junit4]   2> 1257384 T4357 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1257384 T4364 N:127.0.0.1:63147_ C:control_collection S:shard1 C562 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1257384 T4364 N:127.0.0.1:63147_ C:control_collection S:shard1 C562 oasc.SyncStrategy.syncToMe http://127.0.0.1:63147/collection1/ has no replicas
   [junit4]   2> 1257385 T4364 N:127.0.0.1:63147_ C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:63147/collection1/ shard1
   [junit4]   2> 1257385 T4364 N:127.0.0.1:63147_ C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1257385 T4324 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1257385 T4356 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1257385 T4367 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1257386 T4324 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 1257388 T4324 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1257388 T4324 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1257390 T4370 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@b1c1da3 name:ZooKeeperConnection Watcher:127.0.0.1:63380/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1257390 T4324 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1257391 T4356 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1257391 T4324 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1257391 T4357 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63147",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1257394 T4357 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 1257394 T4357 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1257395 T4356 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1257395 T4367 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1257401 T4356 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1257402 T4357 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 1257403 T4356 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1257403 T4367 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1257441 T4364 N:127.0.0.1:63147_ C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:63147/collection1/ and leader is http://127.0.0.1:63147/collection1/
   [junit4]   2> 1257442 T4364 N:127.0.0.1:63147_ C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:63147
   [junit4]   2> 1257442 T4364 N:127.0.0.1:63147_ C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1257442 T4364 N:127.0.0.1:63147_ C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1257442 T4364 N:127.0.0.1:63147_ C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1257444 T4356 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1257445 T4357 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63147",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:63147_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active"} current state version: 4
   [junit4]   2> 1257445 T4357 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63147",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:63147_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active"}
   [junit4]   2> 1257548 T4356 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1257548 T4367 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1258760 T4324 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/shard-1-001/cores/collection1
   [junit4]   2> 1258763 T4324 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/shard-1-001
   [junit4]   2> 1258763 T4324 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1258766 T4324 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:24440
   [junit4]   2> 1258767 T4324 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/shard-1-001/cores, solrconfig=solrconfig.xml, hostPort=24440}
   [junit4]   2> 1258767 T4324 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 1258767 T4324 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/shard-1-001/'
   [junit4]   2> 1258800 T4324 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/shard-1-001/solr.xml
   [junit4]   2> 1258851 T4324 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/shard-1-001/cores
   [junit4]   2> 1258852 T4324 oasc.CoreContainer.<init> New CoreContainer 1513881805
   [junit4]   2> 1258852 T4324 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/shard-1-001/]
   [junit4]   2> 1258852 T4324 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/shard-1-001/lib
   [junit4]   2> 1258853 T4324 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/shard-1-001/lib).
   [junit4]   2> 1258862 T4324 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1258863 T4324 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1258863 T4324 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1258863 T4324 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1258863 T4324 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1258864 T4324 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1258864 T4324 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1258864 T4324 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1258864 T4324 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1258864 T4324 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1258865 T4324 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1258869 T4324 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1258870 T4324 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1258870 T4324 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1258870 T4324 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1258871 T4324 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:63380/solr
   [junit4]   2> 1258871 T4324 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1258871 T4324 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1258872 T4324 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1258873 T4382 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5b06123f name:ZooKeeperConnection Watcher:127.0.0.1:63380 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1258874 T4324 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1258875 T4324 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1258876 T4324 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1258877 T4385 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3eb19aac name:ZooKeeperConnection Watcher:127.0.0.1:63380/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1258877 T4324 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1258880 T4324 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1259884 T4324 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:24440_
   [junit4]   2> 1259885 T4324 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:24440_
   [junit4]   2> 1259889 T4324 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1259891 T4324 N:127.0.0.1:24440_ oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/shard-1-001/cores
   [junit4]   2> 1259893 T4324 N:127.0.0.1:24440_ oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, shard=, transient=false, schema=schema.xml, loadOnStartup=true, name=collection1, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/shard-1-001/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/shard-1-001/cores/collection1/, config=solrconfig.xml, collection=collection1}
   [junit4]   2> 1259894 T4324 N:127.0.0.1:24440_ oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/shard-1-001/cores/collection1/
   [junit4]   2> 1259894 T4324 N:127.0.0.1:24440_ oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1259896 T4386 N:127.0.0.1:24440_ C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1259896 T4386 N:127.0.0.1:24440_ C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1259897 T4356 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1259898 T4386 N:127.0.0.1:24440_ oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 1259898 T4386 N:127.0.0.1:24440_ oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 1259898 T4357 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:24440",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:24440_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down"} current state version: 5
   [junit4]   2> 1259898 T4386 N:127.0.0.1:24440_ oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 1259899 T4357 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:24440",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:24440_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 1259899 T4357 oasco.ReplicaMutator.updateState Collection already exists with numShards=2
   [junit4]   2> 1259899 T4357 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1259899 T4386 N:127.0.0.1:24440_ oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 1259900 T4386 N:127.0.0.1:24440_ oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1259900 T4357 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 1259901 T4385 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1259901 T4385 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 1260900 T4386 N:127.0.0.1:24440_ oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1260901 T4386 N:127.0.0.1:24440_ oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1260901 T4386 N:127.0.0.1:24440_ oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1260902 T4386 N:127.0.0.1:24440_ oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1260902 T4386 N:127.0.0.1:24440_ oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1260902 T4386 N:127.0.0.1:24440_ oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1260920 T4386 N:127.0.0.1:24440_ oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1260923 T4386 N:127.0.0.1:24440_ oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1260933 T4386 N:127.0.0.1:24440_ oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1260956 T4386 N:127.0.0.1:24440_ oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1261008 T4386 N:127.0.0.1:24440_ oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1261109 T4386 N:127.0.0.1:24440_ oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1261120 T4386 N:127.0.0.1:24440_ oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1261567 T4386 N:127.0.0.1:24440_ oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1261583 T4386 N:127.0.0.1:24440_ oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1261586 T4386 N:127.0.0.1:24440_ oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1261597 T4386 N:127.0.0.1:24440_ oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1261602 T4386 N:127.0.0.1:24440_ oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1261606 T4386 N:127.0.0.1:24440_ oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1261607 T4386 N:127.0.0.1:24440_ oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1261607 T4386 N:127.0.0.1:24440_ oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1261607 T4386 N:127.0.0.1:24440_ oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1261608 T4386 N:127.0.0.1:24440_ oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1261608 T4386 N:127.0.0.1:24440_ oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1261608 T4386 N:127.0.0.1:24440_ oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1261608 T4386 N:127.0.0.1:24440_ c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1261609 T4386 N:127.0.0.1:24440_ c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1261609 T4386 N:127.0.0.1:24440_ c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2127155
   [junit4]   2> 1261610 T4386 N:127.0.0.1:24440_ c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1261610 T4386 N:127.0.0.1:24440_ c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1261611 T4386 N:127.0.0.1:24440_ c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1261611 T4386 N:127.0.0.1:24440_ c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.CollectionsAPIDistributedZkTest 336CA9E425036561-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1261612 T4386 N:127.0.0.1:24440_ c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=34, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8097364795430507]
   [junit4]   2> 1261613 T4386 N:127.0.0.1:24440_ c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@7c343fc1 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4a45d79b; maxCacheMB=0.12109375 maxMergeSizeMB=0.826171875)),segFN=segments_1,generation=1}
   [junit4]   2> 1261613 T4386 N:127.0.0.1:24440_ c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1261620 T4386 N:127.0.0.1:24440_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1261620 T4386 N:127.0.0.1:24440_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1261621 T4386 N:127.0.0.1:24440_ c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1261621 T4386 N:127.0.0.1:24440_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1261621 T4386 N:127.0.0.1:24440_ c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1261621 T4386 N:127.0.0.1:24440_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1261622 T4386 N:127.0.0.1:24440_ c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1261622 T4386 N:127.0.0.1:24440_ c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1261622 T4386 N:127.0.0.1:24440_ c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1261644 T4386 N:127.0.0.1:24440_ c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1261646 T4386 N:127.0.0.1:24440_ c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1261649 T4386 N:127.0.0.1:24440_ c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1261651 T4386 N:127.0.0.1:24440_ c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1261659 T4386 N:127.0.0.1:24440_ c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/plugins,/update/csv,/admin/mbeans,/admin/properties,/schema,/update/json,/update/json/docs,/admin/threads,/admin/luke,/admin/file,/admin/segments,standard,/admin/system,/update,/admin/logging,/replication,/config,/admin/ping,/get
   [junit4]   2> 1261662 T4386 N:127.0.0.1:24440_ c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1261663 T4386 N:127.0.0.1:24440_ c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1261663 T4386 N:127.0.0.1:24440_ c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1261665 T4386 N:127.0.0.1:24440_ c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1261666 T4386 N:127.0.0.1:24440_ c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1261667 T4386 N:127.0.0.1:24440_ c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=37, maxMergeAtOnceExplicit=11, maxMergedSegmentMB=51.904296875, floorSegmentMB=0.9609375, forceMergeDeletesPctAllowed=11.590078393234766, segmentsPerTier=41.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6104978885513848
   [junit4]   2> 1261668 T4386 N:127.0.0.1:24440_ c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@7c343fc1 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4a45d79b; maxCacheMB=0.12109375 maxMergeSizeMB=0.826171875)),segFN=segments_1,generation=1}
   [junit4]   2> 1261668 T4386 N:127.0.0.1:24440_ c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1261668 T4386 N:127.0.0.1:24440_ c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6e548a6a[collection1] main
   [junit4]   2> 1261669 T4386 N:127.0.0.1:24440_ c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1261669 T4386 N:127.0.0.1:24440_ c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1261670 T4386 N:127.0.0.1:24440_ c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1261670 T4386 N:127.0.0.1:24440_ c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1261670 T4386 N:127.0.0.1:24440_ c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1261670 T4386 N:127.0.0.1:24440_ c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1261671 T4386 N:127.0.0.1:24440_ c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1261671 T4386 N:127.0.0.1:24440_ c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1261671 T4386 N:127.0.0.1:24440_ c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1261672 T4386 N:127.0.0.1:24440_ c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1261673 T4387 N:127.0.0.1:24440_ c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6e548a6a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1261673 T4386 N:127.0.0.1:24440_ c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1261674 T4390 N:127.0.0.1:24440_ C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:24440 collection:collection1 shard:shard1
   [junit4]   2> 1261674 T4324 N:127.0.0.1:24440_ oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 1261675 T4324 N:127.0.0.1:24440_ oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1261675 T4390 N:127.0.0.1:24440_ C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1261678 T4390 N:127.0.0.1:24440_ C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1261680 T4356 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1261680 T4390 N:127.0.0.1:24440_ C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1261680 T4390 N:127.0.0.1:24440_ C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C563 name=collection1 org.apache.solr.core.SolrCore@7ae6ebc3 url=http://127.0.0.1:24440/collection1 node=127.0.0.1:24440_ C563_STATE=coll:collection1 core:collection1 props:{core=collection1, base_url=http://127.0.0.1:24440, node_name=127.0.0.1:24440_, state=down}
   [junit4]   2> 1261680 T4390 N:127.0.0.1:24440_ C:collection1 S:shard1 C563 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:24440/collection1/
   [junit4]   2> 1261680 T4357 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 1261680 T4390 N:127.0.0.1:24440_ C:collection1 S:shard1 C563 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1261681 T4390 N:127.0.0.1:24440_ C:collection1 S:shard1 C563 oasc.SyncStrategy.syncToMe http://127.0.0.1:24440/collection1/ has no replicas
   [junit4]   2> 1261681 T4390 N:127.0.0.1:24440_ C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:24440/collection1/ shard1
   [junit4]   2> 1261681 T4390 N:127.0.0.1:24440_ C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1261681 T4357 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 1261682 T4385 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1261682 T4385 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 1261684 T4357 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:24440",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 1261685 T4357 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 1261685 T4385 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1261686 T4385 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 1261687 T4356 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1261734 T4390 N:127.0.0.1:24440_ C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:24440/collection1/ and leader is http://127.0.0.1:24440/collection1/
   [junit4]   2> 1261735 T4390 N:127.0.0.1:24440_ C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:24440
   [junit4]   2> 1261735 T4390 N:127.0.0.1:24440_ C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1261735 T4390 N:127.0.0.1:24440_ C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1261735 T4390 N:127.0.0.1:24440_ C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1261737 T4356 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1261738 T4357 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:24440",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:24440_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 1261738 T4357 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	

[...truncated too long message...]

e.java:57)
   [junit4]   2> 		at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 		at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 		at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
   [junit4]   2> 		at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
   [junit4]   2> 		at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
   [junit4]   2> 		at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
   [junit4]   2> 		at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 	
   [junit4]   2> 2073870 T35602 oasc.SolrException.log ERROR Error shutting down core:java.lang.AssertionError: 1
   [junit4]   2> 		at org.apache.solr.core.CachingDirectoryFactory.close(CachingDirectoryFactory.java:201)
   [junit4]   2> 		at org.apache.solr.core.SolrCore.close(SolrCore.java:1176)
   [junit4]   2> 		at org.apache.solr.core.SolrCores.close(SolrCores.java:117)
   [junit4]   2> 		at org.apache.solr.core.CoreContainer.shutdown(CoreContainer.java:378)
   [junit4]   2> 		at org.apache.solr.util.TestHarness.close(TestHarness.java:359)
   [junit4]   2> 		at org.apache.solr.SolrTestCaseJ4.deleteCore(SolrTestCaseJ4.java:704)
   [junit4]   2> 		at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:230)
   [junit4]   2> 		at sun.reflect.GeneratedMethodAccessor32.invoke(Unknown Source)
   [junit4]   2> 		at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 		at java.lang.reflect.Method.invoke(Method.java:606)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:799)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
   [junit4]   2> 		at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 		at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 		at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
   [junit4]   2> 		at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
   [junit4]   2> 		at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
   [junit4]   2> 		at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
   [junit4]   2> 		at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 	
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.request.TestWriterPerf 336CA9E425036561-001
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=DefaultSimilarity, locale=fr_LU, timezone=America/Noronha
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=145835304,total=308805632
   [junit4]   2> NOTE: All tests run in this JVM: [SliceStateTest, TestFoldingMultitermQuery, BadComponentTest, DeleteInactiveReplicaTest, TestSolrQueryParser, URLClassifyProcessorTest, SynonymTokenizerTest, HardAutoCommitTest, TestSuggestSpellingConverter, TestReplicationHandler, TestReplicationHandlerBackup, TestFieldCollectionResource, TestCursorMarkWithoutUniqueKey, TestLazyCores, ChangedSchemaMergeTest, QueryEqualityTest, TestSolr4Spatial2, StatsComponentTest, TestRTGBase, TestBM25SimilarityFactory, DistributedFacetPivotLongTailTest, TestFastWriter, HdfsUnloadDistributedZkTest, TestRemoteStreaming, TestDistributedSearch, TestConfigOverlay, HdfsDirectoryFactoryTest, LukeRequestHandlerTest, TestManagedSynonymFilterFactory, SolrRequestParserTest, SpatialFilterTest, CacheHeaderTest, BlockCacheTest, HdfsWriteToMultipleCollectionsTest, DistributedQueueTest, MigrateRouteKeyTest, TestReversedWildcardFilterFactory, DocumentBuilderTest, InfoHandlerTest, TestConfigReload, ConnectionManagerTest, TestMaxScoreQueryParser, SyncSliceTest, TestFieldTypeResource, TestFieldResource, TestReload, SuggestComponentTest, EnumFieldTest, MergeStrategyTest, SolrIndexSplitterTest, TestBinaryField, TestPerFieldSimilarity, TestHdfsUpdateLog, SOLR749Test, SolrCoreCheckLockOnStartupTest, AnalyticsMergeStrategyTest, TestCSVResponseWriter, TestNRTOpen, IndexBasedSpellCheckerTest, TestWriterPerf]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestWriterPerf -Dtests.seed=336CA9E425036561 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=fr_LU -Dtests.timezone=America/Noronha -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.00s J2 | TestWriterPerf (suite) <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: 1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([336CA9E425036561]:0)
   [junit4]    > 	at org.apache.solr.core.CachingDirectoryFactory.close(CachingDirectoryFactory.java:201)
   [junit4]    > 	at org.apache.solr.core.SolrCore.close(SolrCore.java:1176)
   [junit4]    > 	at org.apache.solr.core.SolrCores.close(SolrCores.java:117)
   [junit4]    > 	at org.apache.solr.core.CoreContainer.shutdown(CoreContainer.java:378)
   [junit4]    > 	at org.apache.solr.util.TestHarness.close(TestHarness.java:359)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.deleteCore(SolrTestCaseJ4.java:704)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:230)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4] Completed [307/482] on J2 in 14.20s, 1 test, 1 failure, 1 error <<< FAILURES!

[...truncated 578 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:543: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:484: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:229: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:511: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1434: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:991: There were test failures: 482 suites, 1914 tests, 1 suite-level error, 2 errors, 1 failure, 60 ignored (10 assumptions)

Total time: 171 minutes 51 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 826111466 bytes
Compression is 0.0%
Took 2 min 32 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 815 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/815/

1 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.handler.TestReplicationHandler

Error Message:
file handle leaks: [FileChannel(/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/index-NIOFSDirectory-010/replication.properties), FileChannel(/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/index-NIOFSDirectory-004/replication.properties)]

Stack Trace:
java.lang.RuntimeException: file handle leaks: [FileChannel(/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/index-NIOFSDirectory-010/replication.properties), FileChannel(/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/index-NIOFSDirectory-004/replication.properties)]
	at __randomizedtesting.SeedInfo.seed([303C02B061F337C5]:0)
	at org.apache.lucene.mockfile.LeakFS.onClose(LeakFS.java:64)
	at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:78)
	at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:79)
	at org.apache.lucene.util.TestRuleTemporaryFilesCleanup.afterAlways(TestRuleTemporaryFilesCleanup.java:212)
	at com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterAlways(TestRuleAdapter.java:31)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.Exception
	at org.apache.lucene.mockfile.LeakFS.onOpen(LeakFS.java:47)
	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
	at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:191)
	at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:204)
	at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:163)
	at java.nio.channels.FileChannel.open(FileChannel.java:287)
	at java.nio.channels.FileChannel.open(FileChannel.java:334)
	at org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:82)
	at org.apache.lucene.util.LuceneTestCase.slowFileExists(LuceneTestCase.java:2487)
	at org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:639)
	at org.apache.solr.handler.ReplicationHandler.loadReplicationProperties(ReplicationHandler.java:977)
	at org.apache.solr.handler.IndexFetcher.logReplicationTimeAndConfFiles(IndexFetcher.java:605)
	at org.apache.solr.handler.IndexFetcher.cleanup(IndexFetcher.java:528)
	at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:518)
	at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:253)
	at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:379)
	at org.apache.solr.handler.ReplicationHandler$2.run(ReplicationHandler.java:1027)
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
	at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 11314 lines...]
   [junit4] Suite: org.apache.solr.handler.TestReplicationHandler
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/init-core-data-001
   [junit4]   2> 1684375 T5460 oas.SolrTestCaseJ4.setUp ###Starting doTestReplicateAfterWrite2Slave
   [junit4]   2> 1684377 T5460 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-001/collection1
   [junit4]   2> 1684396 T5460 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1684400 T5460 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:45418
   [junit4]   2> 1684803 T5460 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=45418, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-001/collection1/data, hostContext=/solr}
   [junit4]   2> 1684804 T5460 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@5fb271f1
   [junit4]   2> 1684805 T5460 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-001/'
   [junit4]   2> 1684861 T5460 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-001/solr.xml
   [junit4]   2> 1684934 T5460 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-001/.
   [junit4]   2> 1684935 T5460 oasc.CoreContainer.<init> New CoreContainer 54381125
   [junit4]   2> 1684935 T5460 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-001/]
   [junit4]   2> 1684936 T5460 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-001/lib
   [junit4]   2> 1684937 T5460 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-001/lib).
   [junit4]   2> 1684952 T5460 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1684952 T5460 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1684952 T5460 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1684953 T5460 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1684953 T5460 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1684953 T5460 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1684954 T5460 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1684954 T5460 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1684954 T5460 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1684955 T5460 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1684955 T5460 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1684966 T5460 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1684968 T5460 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1684968 T5460 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1684969 T5460 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1684971 T5460 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-001/.
   [junit4]   2> 1684974 T5460 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, schema=schema.xml, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-001/./collection1/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-001/./collection1, dataDir=data/, name=collection1, transient=false}
   [junit4]   2> 1684975 T5460 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-001/./collection1/
   [junit4]   2> 1684976 T5460 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1684977 T5470 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-001/./collection1/'
   [junit4]   2> 1685021 T5470 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1685046 T5470 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1685101 T5470 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1685102 T5470 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-001/./collection1/conf/schema.xml
   [junit4]   2> 1685111 T5470 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1685132 T5470 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1685133 T5470 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-001/./collection1/
   [junit4]   2> 1685133 T5470 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1685134 T5470 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-001/./collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-001/./collection1/data/
   [junit4]   2> 1685134 T5470 c:collection1 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 1685136 T5470 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-001/./collection1/data
   [junit4]   2> 1685137 T5470 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-001/./collection1/data/index/
   [junit4]   2> 1685137 T5470 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-001/./collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1685138 T5470 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-001/./collection1/data/index
   [junit4]   2> 1685138 T5470 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=13, maxMergeAtOnceExplicit=11, maxMergedSegmentMB=74.5966796875, floorSegmentMB=2.1953125, forceMergeDeletesPctAllowed=2.047079338343283, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.830014422319695
   [junit4]   2> 1685139 T5470 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@48309204 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6cc269a2),segFN=segments_1,generation=1}
   [junit4]   2> 1685139 T5470 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1685142 T5470 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1685165 T5470 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1685168 T5470 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1685170 T5470 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1685173 T5470 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1685206 T5470 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 1685206 T5470 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /schema,/admin/luke,/config,defaults,/admin/segments,/update/csv,/update,/get,/update/json/docs,/admin/properties,standard,/admin/logging,/admin/threads,/admin/ping,/update/json,lazy,/admin/plugins,/admin/mbeans,/admin/file,/admin/system,/replication
   [junit4]   2> 1685209 T5470 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1685210 T5470 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1685210 T5470 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1685211 T5470 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@2c9e55ad
   [junit4]   2> 1685212 T5470 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@48309204 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6cc269a2),segFN=segments_1,generation=1}
   [junit4]   2> 1685212 T5470 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1685212 T5470 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@54f451d5[collection1] main
   [junit4]   2> 1685213 T5470 c:collection1 oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage initialized to use dir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-001/./collection1/conf
   [junit4]   2> 1685213 T5470 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-001/./collection1/conf}
   [junit4]   2> 1685214 T5470 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using file:dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-001/./collection1/conf
   [junit4]   2> 1685215 T5470 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1685215 T5470 c:collection1 oash.ReplicationHandler.inform Replication enabled for following config files: schema.xml,xslt/dummy.xsl
   [junit4]   2> 1685215 T5470 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1685215 T5471 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@54f451d5[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1685216 T5470 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1685216 T5460 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 1685217 T5460 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1685220 T5460 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-002/collection1
   [junit4]   2> 1685240 T5460 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1685243 T5460 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:45420
   [junit4]   2> 1685243 T5460 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/solr, hostPort=45420, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-002/collection1/data}
   [junit4]   2> 1685244 T5460 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@5fb271f1
   [junit4]   2> 1685244 T5460 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-002/'
   [junit4]   2> 1685301 T5460 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-002/solr.xml
   [junit4]   2> 1685369 T5460 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-002/.
   [junit4]   2> 1685370 T5460 oasc.CoreContainer.<init> New CoreContainer 1570794517
   [junit4]   2> 1685370 T5460 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-002/]
   [junit4]   2> 1685371 T5460 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-002/lib
   [junit4]   2> 1685372 T5460 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-002/lib).
   [junit4]   2> 1685387 T5460 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1685387 T5460 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1685388 T5460 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1685388 T5460 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1685388 T5460 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1685389 T5460 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1685389 T5460 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1685389 T5460 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1685390 T5460 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1685390 T5460 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1685391 T5460 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1685399 T5460 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1685401 T5460 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1685401 T5460 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1685401 T5460 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1685403 T5460 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-002/.
   [junit4]   2> 1685407 T5460 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-002/./collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-002/./collection1/, name=collection1, dataDir=data/, loadOnStartup=true, transient=false, schema=schema.xml, config=solrconfig.xml}
   [junit4]   2> 1685407 T5460 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-002/./collection1/
   [junit4]   2> 1685408 T5460 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1685408 T5482 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-002/./collection1/'
   [junit4]   2> 1685457 T5482 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1685483 T5482 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1685542 T5482 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1685543 T5482 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-002/./collection1/conf/schema.xml
   [junit4]   2> 1685552 T5482 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1685576 T5482 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1685578 T5482 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-002/./collection1/
   [junit4]   2> 1685578 T5482 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1685579 T5482 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-002/./collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-002/./collection1/data/
   [junit4]   2> 1685579 T5482 c:collection1 oasc.SolrCore.<init> JMX monitoring not detected for core: collection1
   [junit4]   2> 1685581 T5482 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-002/./collection1/data
   [junit4]   2> 1685582 T5482 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-002/./collection1/data/index/
   [junit4]   2> 1685583 T5482 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-002/./collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1685583 T5482 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-002/./collection1/data/index
   [junit4]   2> 1685584 T5482 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=13, maxMergeAtOnceExplicit=11, maxMergedSegmentMB=74.5966796875, floorSegmentMB=2.1953125, forceMergeDeletesPctAllowed=2.047079338343283, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.830014422319695
   [junit4]   2> 1685584 T5482 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@2574c523 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@153a900b),segFN=segments_1,generation=1}
   [junit4]   2> 1685585 T5482 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1685587 T5482 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1685617 T5482 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1685620 T5482 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1685623 T5482 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1685626 T5482 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1685629 T5482 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 1685629 T5482 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /schema,/admin/luke,/config,defaults,/admin/segments,/update/csv,/update,/get,/update/json/docs,/admin/properties,standard,/admin/logging,/admin/threads,/admin/ping,/update/json,lazy,/admin/plugins,/admin/mbeans,/admin/file,/admin/system,/replication
   [junit4]   2> 1685632 T5482 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1685633 T5482 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1685633 T5482 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1685634 T5482 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@139b5e16
   [junit4]   2> 1685634 T5482 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@2574c523 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@153a900b),segFN=segments_1,generation=1}
   [junit4]   2> 1685634 T5482 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1685635 T5482 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@318ccb41[collection1] main
   [junit4]   2> 1685636 T5482 c:collection1 oasr.ManagedResourceStorage$FileStorageIO.configure File-based storage initialized to use dir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-002/./collection1/conf
   [junit4]   2> 1685636 T5482 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-002/./collection1/conf}
   [junit4]   2> 1685637 T5482 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using file:dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-002/./collection1/conf
   [junit4]   2> 1685638 T5482 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1685647 T5482 c:collection1 oash.ReplicationHandler.setupPolling Poll Scheduled at an interval of 1000ms
   [junit4]   2> 1685647 T5483 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@318ccb41[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1685648 T5482 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1685648 T5460 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 1685649 T5460 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> ASYNC  NEW_CORE C1009 name=collection1 org.apache.solr.core.SolrCore@754e94a5
   [junit4]   2> 1685653 T5469 c:collection1 C1009 oasc.SolrCore.execute [collection1] webapp=/solr path=/select params={q=*:*&sort=id+desc&wt=javabin&version=2} hits=0 status=0 QTime=1 
   [junit4]   2> 1685658 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[0]} 0 1
   [junit4]   2> 1685661 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[1]} 0 0
   [junit4]   2> 1685664 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[2]} 0 0
   [junit4]   2> 1685667 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[3]} 0 0
   [junit4]   2> 1685671 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[4]} 0 0
   [junit4]   2> 1685674 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[5]} 0 0
   [junit4]   2> 1685677 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[6]} 0 0
   [junit4]   2> 1685680 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[7]} 0 0
   [junit4]   2> 1685684 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[8]} 0 0
   [junit4]   2> 1685688 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[9]} 0 0
   [junit4]   2> 1685691 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[10]} 0 0
   [junit4]   2> 1685695 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[11]} 0 0
   [junit4]   2> 1685698 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[12]} 0 0
   [junit4]   2> 1685701 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[13]} 0 0
   [junit4]   2> 1685704 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[14]} 0 0
   [junit4]   2> 1685707 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[15]} 0 0
   [junit4]   2> 1685710 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[16]} 0 0
   [junit4]   2> 1685713 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[17]} 0 0
   [junit4]   2> 1685716 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[18]} 0 0
   [junit4]   2> 1685719 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[19]} 0 0
   [junit4]   2> 1685722 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[20]} 0 0
   [junit4]   2> 1685725 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[21]} 0 0
   [junit4]   2> 1685728 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[22]} 0 0
   [junit4]   2> 1685732 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[23]} 0 0
   [junit4]   2> 1685735 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[24]} 0 0
   [junit4]   2> 1685738 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[25]} 0 0
   [junit4]   2> 1685742 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[26]} 0 0
   [junit4]   2> 1685746 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[27]} 0 0
   [junit4]   2> 1685749 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[28]} 0 0
   [junit4]   2> 1685752 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[29]} 0 0
   [junit4]   2> 1685755 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[30]} 0 0
   [junit4]   2> 1685758 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[31]} 0 0
   [junit4]   2> 1685761 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[32]} 0 0
   [junit4]   2> 1685766 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[33]} 0 0
   [junit4]   2> 1685770 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[34]} 0 0
   [junit4]   2> 1685774 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[35]} 0 0
   [junit4]   2> 1685777 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[36]} 0 0
   [junit4]   2> 1685781 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[37]} 0 0
   [junit4]   2> 1685785 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[38]} 0 0
   [junit4]   2> 1685789 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[39]} 0 0
   [junit4]   2> 1685792 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[40]} 0 0
   [junit4]   2> 1685796 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[41]} 0 0
   [junit4]   2> 1685800 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[42]} 0 0
   [junit4]   2> 1685803 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[43]} 0 0
   [junit4]   2> 1685806 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[44]} 0 0
   [junit4]   2> 1685809 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[45]} 0 0
   [junit4]   2> 1685813 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[46]} 0 0
   [junit4]   2> 1685817 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[47]} 0 0
   [junit4]   2> 1685820 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[48]} 0 0
   [junit4]   2> 1685824 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[49]} 0 1
   [junit4]   2> 1685827 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[50]} 0 0
   [junit4]   2> 1685830 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[51]} 0 0
   [junit4]   2> 1685833 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[52]} 0 0
   [junit4]   2> 1685836 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[53]} 0 0
   [junit4]   2> 1685839 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[54]} 0 0
   [junit4]   2> 1685842 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[55]} 0 0
   [junit4]   2> 1685846 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[56]} 0 0
   [junit4]   2> 1685848 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[57]} 0 0
   [junit4]   2> 1685851 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[58]} 0 0
   [junit4]   2> 1685855 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[59]} 0 0
   [junit4]   2> 1685858 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[60]} 0 0
   [junit4]   2> 1685861 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[61]} 0 0
   [junit4]   2> 1685864 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[62]} 0 0
   [junit4]   2> 1685867 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[63]} 0 0
   [junit4]   2> 1685870 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[64]} 0 0
   [junit4]   2> 1685874 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[65]} 0 0
   [junit4]   2> 1685877 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[66]} 0 0
   [junit4]   2> 1685881 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[67]} 0 1
   [junit4]   2> 1685885 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[68]} 0 0
   [junit4]   2> 1685888 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[69]} 0 0
   [junit4]   2> 1685892 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[70]} 0 0
   [junit4]   2> 1685895 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[71]} 0 0
   [junit4]   2> 1685899 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[72]} 0 0
   [junit4]   2> 1685902 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[73]} 0 0
   [junit4]   2> 1685906 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[74]} 0 0
   [junit4]   2> 1685910 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[75]} 0 0
   [junit4]   2> 1685914 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[76]} 0 1
   [junit4]   2> 1685917 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[77]} 0 0
   [junit4]   2> 1685920 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[78]} 0 0
   [junit4]   2> 1685923 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[79]} 0 0
   [junit4]   2> 1685926 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[80]} 0 0
   [junit4]   2> 1685930 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[81]} 0 0
   [junit4]   2> 1685933 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[82]} 0 0
   [junit4]   2> 1685936 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[83]} 0 0
   [junit4]   2> 1685939 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[84]} 0 0
   [junit4]   2> 1685942 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[85]} 0 0
   [junit4]   2> 1685945 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[86]} 0 0
   [junit4]   2> 1685948 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[87]} 0 0
   [junit4]   2> 1685951 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[88]} 0 0
   [junit4]   2> 1685954 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[89]} 0 0
   [junit4]   2> 1685957 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[90]} 0 0
   [junit4]   2> 1685961 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[91]} 0 0
   [junit4]   2> 1685964 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[92]} 0 0
   [junit4]   2> 1685967 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[93]} 0 0
   [junit4]   2> 1685970 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[94]} 0 0
   [junit4]   2> 1685973 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[95]} 0 0
   [junit4]   2> 1685976 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[96]} 0 0
   [junit4]   2> 1685979 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[97]} 0 0
   [junit4]   2> 1685982 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[98]} 0 0
   [junit4]   2> 1685985 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[99]} 0 0
   [junit4]   2> 1685988 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[100]} 0 0
   [junit4]   2> 1685991 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[101]} 0 0
   [junit4]   2> 1685994 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[102]} 0 0
   [junit4]   2> 1685997 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[103]} 0 0
   [junit4]   2> 1686000 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[104]} 0 0
   [junit4]   2> 1686003 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[105]} 0 0
   [junit4]   2> 1686006 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[106]} 0 0
   [junit4]   2> 1686009 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[107]} 0 0
   [junit4]   2> 1686012 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[108]} 0 0
   [junit4]   2> 1686015 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[109]} 0 0
   [junit4]   2> 1686018 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[110]} 0 0
   [junit4]   2> 1686021 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[111]} 0 0
   [junit4]   2> 1686024 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[112]} 0 0
   [junit4]   2> 1686027 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[113]} 0 0
   [junit4]   2> 1686030 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[114]} 0 0
   [junit4]   2> 1686033 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[115]} 0 0
   [junit4]   2> 1686036 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[116]} 0 0
   [junit4]   2> 1686039 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[117]} 0 0
   [junit4]   2> 1686042 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[118]} 0 0
   [junit4]   2> 1686045 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[119]} 0 0
   [junit4]   2> 1686049 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[120]} 0 0
   [junit4]   2> 1686052 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[121]} 0 0
   [junit4]   2> 1686057 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[122]} 0 0
   [junit4]   2> 1686061 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[123]} 0 0
   [junit4]   2> 1686064 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[124]} 0 0
   [junit4]   2> 1686067 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[125]} 0 0
   [junit4]   2> 1686070 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[126]} 0 0
   [junit4]   2> 1686073 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[127]} 0 0
   [junit4]   2> 1686076 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[128]} 0 0
   [junit4]   2> 1686079 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[129]} 0 0
   [junit4]   2> 1686083 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[130]} 0 0
   [junit4]   2> 1686086 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[131]} 0 0
   [junit4]   2> 1686089 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[132]} 0 0
   [junit4]   2> 1686093 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[133]} 0 0
   [junit4]   2> 1686096 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[134]} 0 0
   [junit4]   2> 1686099 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[135]} 0 0
   [junit4]   2> 1686102 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[136]} 0 0
   [junit4]   2> 1686105 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[137]} 0 0
   [junit4]   2> 1686108 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[138]} 0 0
   [junit4]   2> 1686111 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[139]} 0 0
   [junit4]   2> 1686114 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[140]} 0 0
   [junit4]   2> 1686117 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[141]} 0 0
   [junit4]   2> 1686121 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[142]} 0 0
   [junit4]   2> 1686125 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[143]} 0 0
   [junit4]   2> 1686128 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[144]} 0 0
   [junit4]   2> 1686131 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[145]} 0 0
   [junit4]   2> 1686134 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[146]} 0 0
   [junit4]   2> 1686137 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[147]} 0 0
   [junit4]   2> 1686141 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[148]} 0 0
   [junit4]   2> 1686146 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[149]} 0 1
   [junit4]   2> 1686146 T5465 c:collection1 C1009 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={qt=/replication&wt=javabin&version=2&command=indexversion} status=0 QTime=0 
   [junit4]   2> 1686149 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[150]} 0 0
   [junit4]   2> 1686152 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[151]} 0 0
   [junit4]   2> 1686155 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[152]} 0 0
   [junit4]   2> 1686158 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[153]} 0 0
   [junit4]   2> 1686161 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[154]} 0 0
   [junit4]   2> 1686164 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[155]} 0 0
   [junit4]   2> 1686168 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[156]} 0 0
   [junit4]   2> 1686171 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[157]} 0 0
   [junit4]   2> 1686175 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[158]} 0 1
   [junit4]   2> 1686178 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[159]} 0 0
   [junit4]   2> 1686181 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[160]} 0 0
   [junit4]   2> 1686184 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[161]} 0 0
   [junit4]   2> 1686188 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[162]} 0 0
   [junit4]   2> 1686191 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[163]} 0 0
   [junit4]   2> 1686194 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[164]} 0 0
   [junit4]   2> 1686197 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[165]} 0 0
   [junit4]   2> 1686200 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[166]} 0 0
   [junit4]   2> 1686203 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[167]} 0 0
   [junit4]   2> 1686207 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[168]} 0 0
   [junit4]   2> 1686210 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[169]} 0 0
   [junit4]   2> 1686213 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[170]} 0 0
   [junit4]   2> 1686217 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[171]} 0 0
   [junit4]   2> 1686220 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[172]} 0 0
   [junit4]   2> 1686223 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[173]} 0 0
   [junit4]   2> 1686226 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[174]} 0 0
   [junit4]   2> 1686229 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[175]} 0 0
   [junit4]   2> 1686240 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[176]} 0 7
   [junit4]   2> 1686244 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[177]} 0 1
   [junit4]   2> 1686247 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[178]} 0 0
   [junit4]   2> 1686250 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[179]} 0 0
   [junit4]   2> 1686253 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[180]} 0 0
   [junit4]   2> 1686256 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[181]} 0 0
   [junit4]   2> 1686259 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[182]} 0 0
   [junit4]   2> 1686263 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[183]} 0 1
   [junit4]   2> 1686266 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[184]} 0 0
   [junit4]   2> 1686270 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[185]} 0 0
   [junit4]   2> 1686275 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[186]} 0 1
   [junit4]   2> 1686279 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[187]} 0 1
   [junit4]   2> 1686282 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[188]} 0 0
   [junit4]   2> 1686285 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[189]} 0 0
   [junit4]   2> 1686289 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[190]} 0 0
   [junit4]   2> 1686293 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[191]} 0 0
   [junit4]   2> 1686296 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[192]} 0 0
   [junit4]   2> 1686301 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[193]} 0 1
   [junit4]   2> 1686305 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[194]} 0 1
   [junit4]   2> 1686309 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[195]} 0 1
   [junit4]   2> 1686314 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[196]} 0 1
   [junit4]   2> 1686318 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[197]} 0 1
   [junit4]   2> 1686323 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[198]} 0 1
   [junit4]   2> 1686326 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[199]} 0 0
   [junit4]   2> 1686329 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[200]} 0 0
   [junit4]   2> 1686332 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[201]} 0 0
   [junit4]   2> 1686336 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[202]} 0 0
   [junit4]   2> 1686340 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[203]} 0 0
   [junit4]   2> 1686344 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[204]} 0 0
   [junit4]   2> 1686349 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[205]} 0 1
   [junit4]   2> 1686352 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[206]} 0 0
   [junit4]   2> 1686356 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[207]} 0 0
   [junit4]   2> 1686359 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[208]} 0 0
   [junit4]   2> 1686362 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[209]} 0 0
   [junit4]   2> 1686366 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[210]} 0 0
   [junit4]   2> 1686371 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[211]} 0 1
   [junit4]   2> 1686374 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[212]} 0 0
   [junit4]   2> 1686379 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[213]} 0 1
   [junit4]   2> 1686383 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[214]} 0 1
   [junit4]   2> 1686386 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[215]} 0 0
   [junit4]   2> 1686389 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[216]} 0 0
   [junit4]   2> 1686392 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[217]} 0 0
   [junit4]   2> 1686395 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[218]} 0 0
   [junit4]   2> 1686398 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[219]} 0 0
   [junit4]   2> 1686402 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[220]} 0 1
   [junit4]   2> 1686405 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[221]} 0 0
   [junit4]   2> 1686408 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[222]} 0 0
   [junit4]   2> 1686413 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[223]} 0 1
   [junit4]   2> 1686416 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[224]} 0 0
   [junit4]   2> 1686419 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[225]} 0 0
   [junit4]   2> 1686422 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[226]} 0 0
   [junit4]   2> 1686425 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[227]} 0 0
   [junit4]   2> 1686428 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[228]} 0 0
   [junit4]   2> 1686432 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[229]} 0 0
   [junit4]   2> 1686435 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[230]} 0 0
   [junit4]   2> 1686438 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[231]} 0 0
   [junit4]   2> 1686441 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[232]} 0 0
   [junit4]   2> 1686444 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[233]} 0 0
   [junit4]   2> 1686448 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[234]} 0 0
   [junit4]   2> 1686451 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[235]} 0 0
   [junit4]   2> 1686454 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[236]} 0 0
   [junit4]   2> 1686457 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[237]} 0 0
   [junit4]   2> 1686460 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[238]} 0 0
   [junit4]   2> 1686463 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[239]} 0 0
   [junit4]   2> 1686466 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[240]} 0 0
   [junit4]   2> 1686469 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[241]} 0 0
   [junit4]   2> 1686473 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[242]} 0 1
   [junit4]   2> 1686476 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[243]} 0 0
   [junit4]   2> 1686480 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[244]} 0 0
   [junit4]   2> 1686484 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[245]} 0 0
   [junit4]   2> 1686487 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[246]} 0 0
   [junit4]   2> 1686490 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[247]} 0 0
   [junit4]   2> 1686493 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[248]} 0 0
   [junit4]   2> 1686496 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[249]} 0 0
   [junit4]   2> 1686499 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[250]} 0 0
   [junit4]   2> 1686502 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[251]} 0 0
   [junit4]   2> 1686505 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[252]} 0 0
   [junit4]   2> 1686509 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[253]} 0 0
   [junit4]   2> 1686512 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[254]} 0 0
   [junit4]   2> 1686515 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[255]} 0 0
   [junit4]   2> 1686518 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[256]} 0 0
   [junit4]   2> 1686521 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[257]} 0 0
   [junit4]   2> 1686524 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[258]} 0 0
   [junit4]   2> 1686528 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[259]} 0 0
   [junit4]   2> 1686531 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[260]} 0 0
   [junit4]   2> 1686534 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[261]} 0 0
   [junit4]   2> 1686538 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[262]} 0 1
   [junit4]   2> 1686542 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[263]} 0 1
   [junit4]   2> 1686545 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[264]} 0 0
   [junit4]   2> 1686548 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[265]} 0 0
   [junit4]   2> 1686551 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[266]} 0 0
   [junit4]   2> 1686554 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[267]} 0 0
   [junit4]   2> 1686557 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[268]} 0 0
   [junit4]   2> 1686560 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[269]} 0 0
   [junit4]   2> 1686565 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[270]} 0 1
   [junit4]   2> 1686568 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[271]} 0 0
   [junit4]   2> 1686571 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[272]} 0 0
   [junit4]   2> 1686574 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[273]} 0 0
   [junit4]   2> 1686578 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[274]} 0 1
   [junit4]   2> 1686581 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[275]} 0 0
   [junit4]   2> 1686586 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[276]} 0 0
   [junit4]   2> 1686591 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[277]} 0 0
   [junit4]   2> 1686594 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[278]} 0 0
   [junit4]   2> 1686598 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[279]} 0 0
   [junit4]   2> 1686601 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[280]} 0 0
   [junit4]   2> 1686605 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[281]} 0 1
   [junit4]   2> 1686608 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[282]} 0 0
   [junit4]   2> 1686612 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[283]} 0 0
   [junit4]   2> 1686615 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[284]} 0 0
   [junit4]   2> 1686618 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[285]} 0 0
   [junit4]   2> 1686621 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[286]} 0 0
   [junit4]   2> 1686625 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[287]} 0 0
   [junit4]   2> 1686630 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[288]} 0 0
   [junit4]   2> 1686634 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[289]} 0 0
   [junit4]   2> 1686638 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[290]} 0 0
   [junit4]   2> 1686642 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[291]} 0 0
   [junit4]   2> 1686646 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[292]} 0 1
   [junit4]   2> 1686650 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[293]} 0 1
   [junit4]   2> 1686653 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[294]} 0 0
   [junit4]   2> 1686656 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[295]} 0 0
   [junit4]   2> 1686659 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[296]} 0 0
   [junit4]   2> 1686664 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[297]} 0 0
   [junit4]   2> 1686669 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[298]} 0 1
   [junit4]   2> 1686672 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[299]} 0 0
   [junit4]   2> 1686676 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[300]} 0 0
   [junit4]   2> 1686679 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[301]} 0 0
   [junit4]   2> 1686682 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[302]} 0 0
   [junit4]   2> 1686685 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[303]} 0 0
   [junit4]   2> 1686688 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[304]} 0 0
   [junit4]   2> 1686691 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[305]} 0 0
   [junit4]   2> 1686693 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[306]} 0 0
   [junit4]   2> 1686696 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[307]} 0 0
   [junit4]   2> 1686699 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[308]} 0 0
   [junit4]   2> 1686704 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[309]} 0 1
   [junit4]   2> 1686707 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[310]} 0 0
   [junit4]   2> 1686711 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[311]} 0 0
   [junit4]   2> 1686714 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[312]} 0 0
   [junit4]   2> 1686717 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[313]} 0 0
   [junit4]   2> 1686720 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[314]} 0 0
   [junit4]   2> 1686723 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[315]} 0 0
   [junit4]   2> 1686726 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[316]} 0 0
   [junit4]   2> 1686729 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[317]} 0 0
   [junit4]   2> 1686733 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[318]} 0 0
   [junit4]   2> 1686736 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[319]} 0 0
   [junit4]   2> 1686739 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[320]} 0 0
   [junit4]   2> 1686742 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[321]} 0 0
   [junit4]   2> 1686745 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[322]} 0 0
   [junit4]   2> 1686748 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[323]} 0 0
   [junit4]   2> 1686752 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[324]} 0 1
   [junit4]   2> 1686755 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[325]} 0 0
   [junit4]   2> 1686758 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[326]} 0 0
   [junit4]   2> 1686761 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[327]} 0 0
   [junit4]   2> 1686764 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[328]} 0 0
   [junit4]   2> 1686767 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[329]} 0 0
   [junit4]   2> 1686770 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[330]} 0 0
   [junit4]   2> 1686773 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[331]} 0 0
   [junit4]   2> 1686776 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[332]} 0 0
   [junit4]   2> 1686779 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[333]} 0 0
   [junit4]   2> 1686783 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[334]} 0 0
   [junit4]   2> 1686786 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[335]} 0 0
   [junit4]   2> 1686790 T5469 c:collection1 C1009 oasup.LogUpdateProcessor.finish [collection1] webapp=/solr path=/update params={wt=javabin&version=2} {add=[336]} 0 1
   [junit4]   2> 1686793 T5469 c:collection1 C1009 oasup.Lo

[...truncated too long message...]

ulative_errors=0}
   [junit4]   2> 2907526 T5460 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 2907526 T5460 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 2907527 T5460 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 2907528 T5460 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 2907529 T5460 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 2907529 T5460 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-028/./collection1/data/index.20150413060227652 [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-028/./collection1/data/index.20150413060227652;done=false>>]
   [junit4]   2> 2907530 T5460 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-028/./collection1/data/index.20150413060227652
   [junit4]   2> 2907530 T5460 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-028/./collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-028/./collection1/data;done=false>>]
   [junit4]   2> 2907530 T5460 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/solr-instance-028/./collection1/data
   [junit4]   2> 2907531 T5460 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 2907728 T5460 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> 1223608 T5459 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50), sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=de, timezone=Asia/Dili
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=37412328,total=243793920
   [junit4]   2> NOTE: All tests run in this JVM: [TestManagedStopFilterFactory, TestStressReorder, TestExactStatsCache, HighlighterConfigTest, RequiredFieldsTest, MinimalSchemaTest, HdfsDirectoryTest, QueryParsingTest, SegmentsInfoRequestHandlerTest, ReplicationFactorTest, TestCollationField, SharedFSAutoReplicaFailoverUtilsTest, SpellCheckCollatorTest, TestMissingGroups, TestMacros, OverseerTest, TestSolrDynamicMBean, DeleteShardTest, OpenExchangeRatesOrgProviderTest, ShardRoutingCustomTest, SimpleFacetsTest, TestImplicitCoreProperties, TestCoreContainer, TestXIncludeConfig, TestCustomSort, TestFastLRUCache, TestRandomFaceting, TestSolrCoreProperties, TestManagedSchemaFieldTypeResource, CloudExitableDirectoryReaderTest, TestUpdate, ParsingFieldUpdateProcessorsTest, ExternalFileFieldSortTest, SolrTestCaseJ4Test, SuggesterTest, CustomCollectionTest, PolyFieldTest, DocValuesMissingTest, TestLRUStatsCache, TestSolrConfigHandler, TimeZoneUtilsTest, RemoteQueryErrorTest, RecoveryZkTest, UUIDFieldTest, SolrIndexConfigTest, DistributedQueryElevationComponentTest, OverseerCollectionProcessorTest, TestDynamicLoading, TestNoOpRegenerator, TestClusterStateMutator, SolrCmdDistributorTest, ZkSolrClientTest, TestManagedResourceStorage, TestSort, TestSolrDeletionPolicy2, HdfsSyncSliceTest, TestFaceting, BasicDistributedZkTest, BasicDistributedZk2Test, UnloadDistributedZkTest, SyncSliceTest, ShardRoutingTest, TestReplicationHandler]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestReplicationHandler -Dtests.seed=303C02B061F337C5 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=de -Dtests.timezone=Asia/Dili -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J0 | TestReplicationHandler (suite) <<<
   [junit4]    > Throwable #1: java.lang.RuntimeException: file handle leaks: [FileChannel(/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/index-NIOFSDirectory-010/replication.properties), FileChannel(/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.handler.TestReplicationHandler 303C02B061F337C5-001/index-NIOFSDirectory-004/replication.properties)]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([303C02B061F337C5]:0)
   [junit4]    > 	at org.apache.lucene.mockfile.LeakFS.onClose(LeakFS.java:64)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:78)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:79)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: java.lang.Exception
   [junit4]    > 	at org.apache.lucene.mockfile.LeakFS.onOpen(LeakFS.java:47)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:191)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:204)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:163)
   [junit4]    > 	at java.nio.channels.FileChannel.open(FileChannel.java:287)
   [junit4]    > 	at java.nio.channels.FileChannel.open(FileChannel.java:334)
   [junit4]    > 	at org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:82)
   [junit4]    > 	at org.apache.lucene.util.LuceneTestCase.slowFileExists(LuceneTestCase.java:2487)
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:639)
   [junit4]    > 	at org.apache.solr.handler.ReplicationHandler.loadReplicationProperties(ReplicationHandler.java:977)
   [junit4]    > 	at org.apache.solr.handler.IndexFetcher.logReplicationTimeAndConfFiles(IndexFetcher.java:605)
   [junit4]    > 	at org.apache.solr.handler.IndexFetcher.cleanup(IndexFetcher.java:528)
   [junit4]    > 	at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:518)
   [junit4]    > 	at org.apache.solr.handler.IndexFetcher.fetchLatestIndex(IndexFetcher.java:253)
   [junit4]    > 	at org.apache.solr.handler.ReplicationHandler.doFetch(ReplicationHandler.java:379)
   [junit4]    > 	at org.apache.solr.handler.ReplicationHandler$2.run(ReplicationHandler.java:1027)
   [junit4]    > 	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
   [junit4]    > 	at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)
   [junit4]    > 	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
   [junit4]    > 	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4] Completed [437/482] on J0 in 1224.17s, 13 tests, 1 error <<< FAILURES!

[...truncated 147 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:543: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:484: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:229: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:510: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1434: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:991: There were test failures: 482 suites, 1914 tests, 1 suite-level error, 60 ignored (10 assumptions)

Total time: 157 minutes 18 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 895703055 bytes
Compression is 0.0%
Took 2 min 38 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 814 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/814/

1 tests failed.
FAILED:  org.apache.solr.cloud.RecoveryZkTest.test

Error Message:
shard1 is not consistent.  Got 862 from http://127.0.0.1:63103/ne_/s/collection1lastClient and got 192 from http://127.0.0.1:63106/ne_/s/collection1

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 862 from http://127.0.0.1:63103/ne_/s/collection1lastClient and got 192 from http://127.0.0.1:63106/ne_/s/collection1
	at __randomizedtesting.SeedInfo.seed([2C3F78376E526F98:A46B47EDC0AE0260]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:57)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 10476 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryZkTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/init-core-data-001
   [junit4]   2> 1381073 T5127 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (true)
   [junit4]   2> 1381074 T5127 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /ne_/s
   [junit4]   2> 1381079 T5127 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1381080 T5128 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1381180 T5127 oasc.ZkTestServer.run start zk server on port:41594
   [junit4]   2> 1381181 T5127 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1381182 T5127 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1381185 T5135 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5e4f0b0 name:ZooKeeperConnection Watcher:127.0.0.1:41594 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1381186 T5127 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1381186 T5127 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1381186 T5127 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1381189 T5127 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1381190 T5127 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1381191 T5138 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@39c60626 name:ZooKeeperConnection Watcher:127.0.0.1:41594/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1381191 T5127 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1381192 T5127 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1381192 T5127 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1381194 T5127 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1381196 T5127 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1381197 T5127 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1381199 T5127 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1381200 T5127 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1381203 T5127 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema15.xml to /configs/conf1/schema.xml
   [junit4]   2> 1381203 T5127 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1381306 T5127 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1381307 T5127 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1381309 T5127 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1381310 T5127 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1381312 T5127 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1381313 T5127 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1381315 T5127 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1381315 T5127 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1381317 T5127 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1381318 T5127 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1381320 T5127 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1381321 T5127 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1381323 T5127 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1381324 T5127 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1381326 T5127 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1381326 T5127 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1381328 T5127 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1381329 T5127 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1382587 T5127 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/control-001/cores/collection1
   [junit4]   2> 1382591 T5127 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1382596 T5127 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:63098
   [junit4]   2> 1382596 T5127 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=63098, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/control-001/cores, hostContext=/ne_/s, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/tempDir-001/control/data}
   [junit4]   2> 1382597 T5127 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 1382597 T5127 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/control-001/'
   [junit4]   2> 1382645 T5127 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/control-001/solr.xml
   [junit4]   2> 1382709 T5127 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/control-001/cores
   [junit4]   2> 1382709 T5127 oasc.CoreContainer.<init> New CoreContainer 972999988
   [junit4]   2> 1382710 T5127 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/control-001/]
   [junit4]   2> 1382710 T5127 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/control-001/lib
   [junit4]   2> 1382711 T5127 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/control-001/lib).
   [junit4]   2> 1382724 T5127 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1382725 T5127 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1382725 T5127 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1382725 T5127 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1382726 T5127 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1382726 T5127 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1382726 T5127 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1382726 T5127 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1382727 T5127 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1382727 T5127 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1382727 T5127 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1382734 T5127 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1382735 T5127 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1382736 T5127 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1382736 T5127 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1382737 T5127 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:41594/solr
   [junit4]   2> 1382737 T5127 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1382737 T5127 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1382738 T5127 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1382740 T5150 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5679af85 name:ZooKeeperConnection Watcher:127.0.0.1:41594 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1382741 T5127 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1382741 T5127 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1382743 T5127 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1382744 T5153 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@13ad131f name:ZooKeeperConnection Watcher:127.0.0.1:41594/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1382745 T5127 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1382746 T5127 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1382748 T5127 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1382751 T5127 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1382753 T5127 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1382755 T5127 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1382757 T5127 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1382759 T5127 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1382760 T5127 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1382762 T5127 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63098_ne_%2Fs
   [junit4]   2> 1382762 T5127 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:63098_ne_%2Fs
   [junit4]   2> 1382764 T5127 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1382766 T5127 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1382767 T5127 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1382769 T5127 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:63098_ne_%2Fs
   [junit4]   2> 1382769 T5127 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1382771 T5127 oasc.Overseer.start Overseer (id=93635670365437955-127.0.0.1:63098_ne_%2Fs-n_0000000000) starting
   [junit4]   2> 1382773 T5127 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1382780 T5127 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1382780 T5155 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1382780 T5127 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1382781 T5154 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1382784 T5127 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/control-001/cores
   [junit4]   2> 1382786 T5127 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/control-001/cores/collection1, loadOnStartup=true, collection=control_collection, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/control-001/cores/collection1/, coreNodeName=, config=solrconfig.xml, name=collection1, transient=false, dataDir=data/, schema=schema.xml}
   [junit4]   2> 1382786 T5127 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/control-001/cores/collection1/
   [junit4]   2> 1382787 T5127 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1382788 T5157 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1382788 T5157 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1382789 T5153 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1382789 T5157 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1382790 T5154 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:63098_ne_%2Fs",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63098/ne_/s",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"} current state version: 0
   [junit4]   2> 1382790 T5154 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:63098_ne_%2Fs",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63098/ne_/s",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 1382790 T5154 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1382791 T5154 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1382792 T5153 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1383789 T5157 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1383790 T5157 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1383791 T5157 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1383791 T5157 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1383792 T5157 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1383792 T5157 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/control-001/cores/collection1/'
   [junit4]   2> 1383809 T5157 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1383812 T5157 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1383823 T5157 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1383845 T5157 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1383895 T5157 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1383996 T5157 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1384006 T5157 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1384385 T5157 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1384388 T5157 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1384390 T5157 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1384395 T5157 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1384414 T5157 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1384415 T5157 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1384415 T5157 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/control-001/cores/collection1/data/
   [junit4]   2> 1384416 T5157 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1fe167f3
   [junit4]   2> 1384417 T5157 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/control-001/cores/collection1/data
   [junit4]   2> 1384417 T5157 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1384417 T5157 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1384418 T5157 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/control-001/cores/collection1/data/index
   [junit4]   2> 1384418 T5157 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=87.6728515625, floorSegmentMB=1.197265625, forceMergeDeletesPctAllowed=15.735976729135663, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2923950719088899
   [junit4]   2> 1384419 T5157 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@439a72fa lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@50f29dc2),segFN=segments_1,generation=1}
   [junit4]   2> 1384419 T5157 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1384425 T5157 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1384425 T5157 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1384426 T5157 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1384426 T5157 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1384426 T5157 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1384426 T5157 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1384427 T5157 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1384427 T5157 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1384427 T5157 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1384447 T5157 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1384449 T5157 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1384452 T5157 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1384454 T5157 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1384459 T5157 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/ping,/admin/luke,/replication,/admin/logging,/update/csv,/update/json,/update,standard,/admin/mbeans,/admin/plugins,/admin/properties,/admin/threads,/config,/update/json/docs,/admin/system,/admin/file,/admin/segments,/get,/schema
   [junit4]   2> 1384461 T5157 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1384463 T5157 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1384463 T5157 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1384465 T5157 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1384465 T5157 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1384466 T5157 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1521501496, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1384466 T5157 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@439a72fa lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@50f29dc2),segFN=segments_1,generation=1}
   [junit4]   2> 1384467 T5157 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1384467 T5157 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@54a7d149[collection1] main
   [junit4]   2> 1384467 T5157 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1384468 T5157 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1384469 T5157 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1384469 T5157 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1384469 T5157 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1384470 T5157 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1384470 T5157 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1384470 T5157 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1384471 T5157 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1384471 T5157 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1384472 T5158 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@54a7d149[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1384472 T5157 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1384473 T5161 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:63098/ne_/s collection:control_collection shard:shard1
   [junit4]   2> 1384474 T5127 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 1384474 T5127 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1384474 T5161 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1384477 T5127 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1384478 T5127 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1384479 T5161 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1384480 T5164 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@680954cd name:ZooKeeperConnection Watcher:127.0.0.1:41594/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1384480 T5127 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1384481 T5127 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1384481 T5153 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1384481 T5161 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1384481 T5127 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1384482 T5161 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1384482 T5154 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C752 name=collection1 org.apache.solr.core.SolrCore@55d51534 url=http://127.0.0.1:63098/ne_/s/collection1 node=127.0.0.1:63098_ne_%2Fs C752_STATE=coll:control_collection core:collection1 props:{core=collection1, node_name=127.0.0.1:63098_ne_%2Fs, state=down, base_url=http://127.0.0.1:63098/ne_/s}
   [junit4]   2> 1384483 T5161 C:control_collection S:shard1 C752 P63098 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:63098/ne_/s/collection1/
   [junit4]   2> 1384483 T5161 C:control_collection S:shard1 C752 P63098 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1384484 T5161 C:control_collection S:shard1 C752 P63098 oasc.SyncStrategy.syncToMe http://127.0.0.1:63098/ne_/s/collection1/ has no replicas
   [junit4]   2> 1384484 T5161 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:63098/ne_/s/collection1/ shard1
   [junit4]   2> 1384484 T5164 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 1384484 T5153 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1384484 T5161 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1384486 T5127 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1384486 T5127 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 1384486 T5127 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1384487 T5127 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1384489 T5167 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@113d238b name:ZooKeeperConnection Watcher:127.0.0.1:41594/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1384489 T5127 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1384490 T5127 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1384490 T5153 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1384492 T5154 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63098/ne_/s",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1384495 T5154 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 1384495 T5154 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1384496 T5153 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1384496 T5164 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1384500 T5153 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1384500 T5154 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 1384502 T5153 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1384502 T5164 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1384541 T5161 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:63098/ne_/s/collection1/ and leader is http://127.0.0.1:63098/ne_/s/collection1/
   [junit4]   2> 1384542 T5161 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:63098/ne_/s
   [junit4]   2> 1384542 T5161 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1384542 T5161 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1384543 T5161 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1384544 T5153 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1384545 T5154 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:63098_ne_%2Fs",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63098/ne_/s",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1"} current state version: 4
   [junit4]   2> 1384546 T5154 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:63098_ne_%2Fs",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63098/ne_/s",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 1384650 T5164 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1384650 T5153 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1385914 T5127 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-1-001/cores/collection1
   [junit4]   2> 1385916 T5127 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-1-001
   [junit4]   2> 1385917 T5127 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1385919 T5127 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:63103
   [junit4]   2> 1385920 T5127 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=63103, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/tempDir-001/jetty1, hostContext=/ne_/s}
   [junit4]   2> 1385920 T5127 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 1385920 T5127 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-1-001/'
   [junit4]   2> 1385952 T5127 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-1-001/solr.xml
   [junit4]   2> 1385988 T5127 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-1-001/cores
   [junit4]   2> 1385988 T5127 oasc.CoreContainer.<init> New CoreContainer 650824738
   [junit4]   2> 1385989 T5127 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-1-001/]
   [junit4]   2> 1385989 T5127 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-1-001/lib
   [junit4]   2> 1385990 T5127 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-1-001/lib).
   [junit4]   2> 1386000 T5127 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1386001 T5127 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1386001 T5127 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1386001 T5127 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1386002 T5127 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1386002 T5127 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1386002 T5127 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1386002 T5127 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1386002 T5127 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1386003 T5127 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1386003 T5127 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1386008 T5127 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1386008 T5127 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1386009 T5127 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1386009 T5127 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1386009 T5127 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:41594/solr
   [junit4]   2> 1386009 T5127 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1386010 T5127 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1386010 T5127 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1386012 T5179 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@658fbf04 name:ZooKeeperConnection Watcher:127.0.0.1:41594 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1386013 T5127 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1386014 T5127 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1386016 T5127 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1386017 T5182 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4a81eb40 name:ZooKeeperConnection Watcher:127.0.0.1:41594/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1386017 T5127 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1386021 T5127 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1387027 T5127 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63103_ne_%2Fs
   [junit4]   2> 1387028 T5127 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:63103_ne_%2Fs
   [junit4]   2> 1387030 T5127 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1387033 T5127 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-1-001/cores
   [junit4]   2> 1387035 T5127 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-1-001/cores/collection1/, coreNodeName=, shard=, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-1-001/cores/collection1, name=collection1, dataDir=data/, collection=collection1, transient=false, schema=schema.xml, loadOnStartup=true}
   [junit4]   2> 1387035 T5127 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-1-001/cores/collection1/
   [junit4]   2> 1387036 T5127 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1387038 T5183 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1387038 T5183 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1387039 T5153 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1387040 T5183 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 1387040 T5183 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 1387040 T5154 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:63103_ne_%2Fs",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63103/ne_/s",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"} current state version: 5
   [junit4]   2> 1387041 T5183 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 1387041 T5154 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:63103_ne_%2Fs",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63103/ne_/s",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 1387041 T5154 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 1387042 T5154 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1387042 T5183 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 1387042 T5183 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1387043 T5154 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 1387043 T5182 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1387044 T5182 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 1388042 T5183 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1388043 T5183 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1388044 T5183 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1388044 T5183 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1388045 T5183 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1388045 T5183 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1388062 T5183 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1388065 T5183 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1388076 T5183 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1388100 T5183 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1388149 T5183 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1388250 T5183 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1388260 T5183 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1388629 T5183 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1388633 T5183 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1388635 T5183 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1388640 T5183 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1388661 T5183 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1388661 T5183 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1388661 T5183 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1388662 T5183 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1fe167f3
   [junit4]   2> 1388662 T5183 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1388663 T5183 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1388663 T5183 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1388664 T5183 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1388665 T5183 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=87.6728515625, floorSegmentMB=1.197265625, forceMergeDeletesPctAllowed=15.735976729135663, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2923950719088899
   [junit4]   2> 1388666 T5183 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6de49f01 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3c833304),segFN=segments_1,generation=1}
   [junit4]   2> 1388666 T5183 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1388672 T5183 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1388673 T5183 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1388673 T5183 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1388673 T5183 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1388674 T5183 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1388674 T5183 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1388674 T5183 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1388675 T5183 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1388675 T5183 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1388703 T5183 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1388706 T5183 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1388709 T5183 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1388711 T5183 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1388720 T5183 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/ping,/admin/luke,/replication,/admin/logging,/update/csv,/update/json,/update,standard,/admin/mbeans,/admin/plugins,/admin/properties,/admin/threads,/config,/update/json/docs,/admin/system,/admin/file,/admin/segments,/get,/schema
   [junit4]   2> 1388722 T5183 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1388724 T5183 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1388724 T5183 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1388726 T5183 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1388727 T5183 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1388728 T5183 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1521501496, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1388729 T5183 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6de49f01 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3c833304),segFN=segments_1,generation=1}
   [junit4]   2> 1388729 T5183 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1388730 T5183 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4f76fe19[collection1] main
   [junit4]   2> 1388730 T5183 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1388731 T5183 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1388731 T5183 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1388732 T5183 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1388732 T5183 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1388732 T5183 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1388733 T5183 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1388733 T5183 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1388733 T5183 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1388734 T5183 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1388735 T5184 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4f76fe19[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1388735 T5183 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1388736 T5187 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:63103/ne_/s collection:collection1 shard:shard1
   [junit4]   2> 1388736 T5127 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 1388737 T5127 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1388737 T5187 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1388742 T5187 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1388744 T5153 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1388744 T5187 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1388744 T5187 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C753 name=collection1 org.apache.solr.core.SolrCore@3cfd6af url=http://127.0.0.1:63103/ne_/s/collection1 node=127.0.0.1:63103_ne_%2Fs C753_STATE=coll:collection1 core:collection1 props:{core=collection1, node_name=127.0.0.1:63103_ne_%2Fs, state=down, base_url=http://127.0.0.1:63103/ne_/s}
   [junit4]   2> 1388745 T5187 C:collection1 S:shard1 C753 P63103 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:63103/ne_/s/collection1/
   [junit4]   2> 1388745 T5154 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 1388745 T5187 C:collection1 S:shard1 C753 P63103 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1388746 T5187 C:collection1 S:shard1 C753 P63103 oasc.SyncStrategy.syncToMe http://127.0.0.1:63103/ne_/s/collection1/ has no replicas
   [junit4]   2> 1388746 T5187 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:63103/ne_/s/collection1/ shard1
   [junit4]   2> 1388746 T5187 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1388746 T5154 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 1388747 T5182 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1388748 T5182 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 1388752 T5153 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1388753 T5154 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63103/ne_/s",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 1388754 T5154 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 1388755 T5182 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1388756 T5182 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 1388803 T5187 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:63103/ne_/s/collection1/ and leader is http://127.0.0.1:63103/ne_/s/collection1/
   [junit4]   2> 1388804 T5187 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:63103/ne_/s
   [junit4]   2> 1388804 T5187 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1388804 T5187 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1388804 T5187 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1388806 T5153 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1388807 T5154 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:63103_ne_%2Fs",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63103/ne_/s",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1"} current state version: 5
   [junit4]   2> 1388807 T5154 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:63103_ne_%2Fs",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63103/ne_/s",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 1388808 T5187 C:collection1 S:shard1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 1388809 T5154 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 3
   [junit4]   2> 1388809 T5182 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1388810 T5182 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 1390641 T5127 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-2-001/cores/collection1
   [junit4]   2> 1390644 T5127 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-2-001
   [junit4]   2> 1390645 T5127 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1390647 T5127 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:63106
   [junit4]   2> 1390648 T5127 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/tempDir-001/jetty2, solrconfig=solrconfig.xml, hostContext=/ne_/s, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-2-001/cores, hostPort=63106}
   [junit4]   2> 1390648 T5127 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 1390648 T5127 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-2-001/'
   [junit4]   2> 1390680 T5127 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-2-001/solr.xml
   [junit4]   2> 1390746 T5127 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-2-001/cores
   [junit4]   2> 1390746 T5127 oasc.CoreContainer.<init> New CoreContainer 507398925
   [junit4]   2> 1390747 T5127 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-2-001/]
   [junit4]   2> 1390747 T5127 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-2-001/lib
   [junit4]   2> 1390748 T5127 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-2-001/lib).
   [junit4]   2> 1390761 T5127 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1390761 T5127 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1390762 T5127 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1390762 T5127 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1390762 T5127 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1390763 T5127 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1390763 T5127 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1390763 T5127 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1390764 T5127 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1390764 T5127 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1390764 T5127 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1390771 T5127 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1390772 T5127 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1390773 T5127 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1390773 T5127 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1390773 T5127 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:41594/solr
   [junit4]   2> 1390774 T5127 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1390774 T5127 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1390775 T5127 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1390777 T5199 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@389ed5 name:ZooKeeperConnection Watcher:127.0.0.1:41594 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1390777 T5127 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1390778 T5127 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1390780 T5127 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1390781 T5202 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@77ba6427 name:ZooKeeperConnection Watcher:127.0.0.1:41594/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1390782 T5127 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1390787 T5127 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1391793 T5127 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:63106_ne_%2Fs
   [junit4]   2> 1391794 T5127 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:63106_ne_%2Fs
   [junit4]   2> 1391797 T5127 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1391799 T5127 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-2-001/cores
   [junit4]   2> 1391801 T5127 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, name=collection1, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-2-001/cores/collection1, config=solrconfig.xml, dataDir=data/, schema=schema.xml, shard=, transient=false, collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-2-001/cores/collection1/}
   [junit4]   2> 1391802 T5127 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-2-001/cores/collection1/
   [junit4]   2> 1391802 T5127 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1391804 T5203 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1391805 T5203 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1391806 T5153 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1391807 T5203 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 1391807 T5154 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:63106_ne_%2Fs",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63106/ne_/s",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"} current state version: 5
   [junit4]   2> 1391807 T5203 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 1391808 T5154 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:63106_ne_%2Fs",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:63106/ne_/s",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 1391808 T5203 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 1391808 T5154 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 1391809 T5154 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1391809 T5203 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 1391810 T5203 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1391810 T5154 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 4
   [junit4]   2> 1391810 T5182 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1391810 T5202 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 3)
   [junit4]   2> 1391811 T5182 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 5 
   [junit4]   2> 1391812 T5202 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 5 
   [junit4]   2> 1392810 T5203 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1392811 T5203 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1392811 T5203 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1392812 T5203 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1392812 T5203 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1392813 T5203 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-2-001/cores/collection1/'
   [junit4]   2> 1392830 T5203 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1392833 T5203 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1392844 T5203 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1392866 T5203 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1392916 T5203 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1393017 T5203 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1393027 T5203 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1393382 T5203 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1393385 T5203 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1393387 T5203 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1393392 T5203 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1393415 T5203 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1393415 T5203 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1393416 T5203 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-2-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-2-001/cores/collection1/data/
   [junit4]   2> 1393416 T5203 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1fe167f3
   [junit4]   2> 1393417 T5203 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-2-001/cores/collection1/data
   [junit4]   2> 1393418 T5203 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-2-001/cores/collection1/data/index/
   [junit4]   2> 1393418 T5203 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-2-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1393419 T5203 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-2-001/cores/collection1/data/index
   [junit4]   2> 1393419 T5203 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=28, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=87.6728515625, floorSegmentMB=1.197265625, forceMergeDeletesPctAllowed=15.735976729135663, segmentsPerTier=36.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.2923950719088899
   [junit4]   2> 1393420 T5203 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2ba68443 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@41356118),segFN=segments_1,generation=1}
   [junit4]   2> 1393420 T5203 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits 

[...truncated too long message...]

joinLeaderElection Not rejoining election because CoreContainer is closed
   [junit4]   2> 1553027 T5245 C:collection1 S:shard1 c:collection1 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@18c83cda
   [junit4]   2> 1553028 T5245 C:collection1 S:shard1 c:collection1 oasc.ZkController.unRegisterConfListener  a listener was removed because of core close
   [junit4]   2> 1553029 T5245 C:collection1 S:shard1 c:collection1 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=63,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=351,cumulative_deletesById=167,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=32882,transaction_logs_total_number=1}
   [junit4]   2> 1553029 T5245 C:collection1 S:shard1 c:collection1 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 1553029 T5245 C:collection1 S:shard1 c:collection1 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 1553029 T5245 C:collection1 S:shard1 c:collection1 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 1553030 T5245 C:collection1 S:shard1 c:collection1 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 1553037 T5245 C:collection1 S:shard1 c:collection1 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 1553037 T5245 C:collection1 S:shard1 c:collection1 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-2-001/cores/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-2-001/cores/collection1/data;done=false>>]
   [junit4]   2> 1553037 T5245 C:collection1 S:shard1 c:collection1 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-2-001/cores/collection1/data
   [junit4]   2> 1553037 T5245 C:collection1 S:shard1 c:collection1 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-2-001/cores/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-2-001/cores/collection1/data/index;done=false>>]
   [junit4]   2> 1553038 T5245 C:collection1 S:shard1 c:collection1 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001/shard-2-001/cores/collection1/data/index
   [junit4]   2> 1553039 T5127 C:collection1 S:shard1 c:collection1 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/ne_/s,null}
   [junit4]   2> 1553242 T5127 C:collection1 S:shard1 c:collection1 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:41594 41594
   [junit4]   2> 1553387 T5128 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:41594 41594
   [junit4]   2> 1553388 T5128 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		5	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/aliases.json
   [junit4]   2> 		4	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/overseer_elect/election/93635670365437959-127.0.0.1:63103_ne_%2Fs-n_0000000001
   [junit4]   2> 		2	/solr/collections/collection1/leader_elect/shard1/election/93635670365437959-core_node1-n_0000000000
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryZkTest -Dtests.method=test -Dtests.seed=2C3F78376E526F98 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=sq -Dtests.timezone=Africa/Banjul -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] FAILURE  172s J0 | RecoveryZkTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: shard1 is not consistent.  Got 862 from http://127.0.0.1:63103/ne_/s/collection1lastClient and got 192 from http://127.0.0.1:63106/ne_/s/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([2C3F78376E526F98:A46B47EDC0AE0260]:0)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:123)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 1553405 T5127 C:collection1 S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.RecoveryZkTest 2C3F78376E526F98-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {id=Lucene50(blocksize=128), a_i=Lucene50(blocksize=128), a_t=PostingsFormat(name=Direct), rnd_b=Lucene50(blocksize=128), _version_=FST50}, docValues:{}, sim=DefaultSimilarity, locale=sq, timezone=Africa/Banjul
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=70845808,total=332398592
   [junit4]   2> NOTE: All tests run in this JVM: [DistributedQueryElevationComponentTest, DistributedFacetPivotSmallAdvancedTest, TestRandomFaceting, TestDynamicLoading, HdfsBasicDistributedZk2Test, SolrCmdDistributorTest, TestSolrIndexConfig, XmlUpdateRequestHandlerTest, SegmentsInfoRequestHandlerTest, DeleteShardTest, TestTrie, TestPartialUpdateDeduplication, TestCloudInspectUtil, SpellCheckCollatorTest, DistributedSpellCheckComponentTest, HighlighterConfigTest, OpenExchangeRatesOrgProviderTest, TestDefaultSearchFieldResource, TestSort, SolrTestCaseJ4Test, TestUpdate, MBeansHandlerTest, RequiredFieldsTest, DistributedFacetPivotWhiteBoxTest, TestIBSimilarityFactory, TestSolrDeletionPolicy2, OverseerTest, SolrIndexConfigTest, TestSolrCoreProperties, ShowFileRequestHandlerTest, TestNoOpRegenerator, SharedFSAutoReplicaFailoverUtilsTest, CursorPagingTest, DebugComponentTest, MinimalSchemaTest, DistanceFunctionTest, ZkNodePropsTest, TestXIncludeConfig, TestExactStatsCache, ExternalFileFieldSortTest, TestFaceting, TestDFRSimilarityFactory, StressHdfsTest, TestLRUStatsCache, DocValuesMissingTest, SignatureUpdateProcessorFactoryTest, QueryParsingTest, OverseerCollectionProcessorTest, StatelessScriptUpdateProcessorFactoryTest, TestManagedResourceStorage, DOMUtilTest, TestManagedStopFilterFactory, HdfsRecoveryZkTest, TestMissingGroups, DateRangeFieldTest, TestMiniSolrCloudClusterSSL, ReplicationFactorTest, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, LeaderElectionIntegrationTest, RecoveryZkTest]
   [junit4] Completed [191/482] on J0 in 172.37s, 1 test, 1 failure <<< FAILURES!

[...truncated 939 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:543: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:484: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:229: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:510: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1434: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:991: There were test failures: 482 suites, 1914 tests, 1 failure, 59 ignored (9 assumptions)

Total time: 158 minutes 23 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 915691096 bytes
Compression is 0.0%
Took 2 min 45 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 813 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/813/

1 tests failed.
REGRESSION:  org.apache.lucene.analysis.core.TestRandomChains.testRandomChains

Error Message:
startOffset must be non-negative, and endOffset must be >= startOffset, startOffset=11,endOffset=10

Stack Trace:
java.lang.IllegalArgumentException: startOffset must be non-negative, and endOffset must be >= startOffset, startOffset=11,endOffset=10
	at __randomizedtesting.SeedInfo.seed([E064DFCFBDF444E4:DD85F6AEFAE65924]:0)
	at org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl.setOffset(PackedTokenAttributeImpl.java:107)
	at org.apache.lucene.analysis.shingle.ShingleFilter.incrementToken(ShingleFilter.java:345)
	at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:68)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:700)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:611)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:509)
	at org.apache.lucene.analysis.core.TestRandomChains.testRandomChains(TestRandomChains.java:909)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 2968 lines...]
   [junit4] Suite: org.apache.lucene.analysis.core.TestRandomChains
   [junit4]   2> TEST FAIL: useCharFilter=false text='h\u030a\ue4c1\u5b0f&\udba4\ude2cS \ud7f4\ud7c2\ud7bc\ud7d3\ud7b9  \"&'
   [junit4]   2> Exception from random analyzer: 
   [junit4]   2> charfilters=
   [junit4]   2>   org.apache.lucene.analysis.charfilter.MappingCharFilter(org.apache.lucene.analysis.charfilter.NormalizeCharMap@34c2d068, java.io.StringReader@620076e7)
   [junit4]   2> tokenizer=
   [junit4]   2>   org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer(org.apache.lucene.util.AttributeFactory$1@4d143cb7)
   [junit4]   2> filters=
   [junit4]   2>   org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter(ValidatingTokenFilter@4b75f7fd term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word,flags=0,payload=null, true)
   [junit4]   2>   org.apache.lucene.analysis.de.GermanMinimalStemFilter(ValidatingTokenFilter@6ba8d226 term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word,flags=0,payload=null,keyword=false)
   [junit4]   2>   org.apache.lucene.analysis.cjk.CJKBigramFilter(ValidatingTokenFilter@3377bbdd term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word,flags=0,payload=null,keyword=false, -3, true)
   [junit4]   2>   org.apache.lucene.analysis.shingle.ShingleFilter(ValidatingTokenFilter@75cb0869 term=,bytes=[],startOffset=0,endOffset=0,positionIncrement=1,positionLength=1,type=word,flags=0,payload=null,keyword=false, 4, 8)
   [junit4]   2> offsetsAreCorrect=false
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestRandomChains -Dtests.method=testRandomChains -Dtests.seed=E064DFCFBDF444E4 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=hu_HU -Dtests.timezone=Africa/Algiers -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   66.9s J0 | TestRandomChains.testRandomChains <<<
   [junit4]    > Throwable #1: java.lang.IllegalArgumentException: startOffset must be non-negative, and endOffset must be >= startOffset, startOffset=11,endOffset=10
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([E064DFCFBDF444E4:DD85F6AEFAE65924]:0)
   [junit4]    > 	at org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl.setOffset(PackedTokenAttributeImpl.java:107)
   [junit4]    > 	at org.apache.lucene.analysis.shingle.ShingleFilter.incrementToken(ShingleFilter.java:345)
   [junit4]    > 	at org.apache.lucene.analysis.ValidatingTokenFilter.incrementToken(ValidatingTokenFilter.java:68)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkAnalysisConsistency(BaseTokenStreamTestCase.java:700)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:611)
   [junit4]    > 	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:509)
   [junit4]    > 	at org.apache.lucene.analysis.core.TestRandomChains.testRandomChains(TestRandomChains.java:909)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/analysis/common/test/J0/temp/lucene.analysis.core.TestRandomChains E064DFCFBDF444E4-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {dummy=Lucene50(blocksize=128)}, docValues:{}, sim=DefaultSimilarity, locale=hu_HU, timezone=Africa/Algiers
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=96506288,total=273678336
   [junit4]   2> NOTE: All tests run in this JVM: [TestFactories, TestItalianLightStemFilterFactory, TestCharArraySet, TestTrimFilterFactory, TestGermanNormalizationFilter, TestFrenchMinimalStemFilterFactory, TestCommonGramsQueryFilterFactory, TestDutchAnalyzer, TestPortugueseMinimalStemFilter, TestMorph, TestKeywordAnalyzer, TestReverseStringFilter, Test64kAffixes, TestCaseSensitive, TestLatvianStemmer, TestTurkishLowerCaseFilter, TestCaseInsensitive, TestBugInSomething, TestHyphenationCompoundWordTokenFilterFactory, TestUAX29URLEmailAnalyzer, TestCollationKeyAnalyzer, TestPorterStemFilterFactory, TestIrishLowerCaseFilter, TestPathHierarchyTokenizer, TestFlagNum, TestStemmerOverrideFilterFactory, TestScandinavianNormalizationFilter, TestIndonesianStemmer, TestArmenianAnalyzer, TestBulgarianAnalyzer, QueryAutoStopWordAnalyzerTest, TestBrazilianAnalyzer, TestPatternCaptureGroupTokenFilter, TestSnowball, TestRandomChains]
   [junit4] Completed [135/271] on J0 in 106.66s, 2 tests, 1 error <<< FAILURES!

[...truncated 414 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:543: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:484: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:457: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:2230: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/analysis/build.xml:106: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/analysis/build.xml:38: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/module-build.xml:58: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1434: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:991: There were test failures: 271 suites, 1425 tests, 1 error, 1 ignored

Total time: 95 minutes 28 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 1728310 bytes
Compression is 0.0%
Took 0.58 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 812 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/812/

2 tests failed.
REGRESSION:  org.apache.lucene.facet.taxonomy.TestSearcherTaxonomyManager.testDirectory

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([39FAAAC972D62F74]:0)


FAILED:  junit.framework.TestSuite.org.apache.lucene.facet.taxonomy.TestSearcherTaxonomyManager

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([39FAAAC972D62F74]:0)




Build Log:
[...truncated 7355 lines...]
   [junit4] Suite: org.apache.lucene.facet.taxonomy.TestSearcherTaxonomyManager
   [junit4]   2> ??? 09, 2015 11:12:48 AM com.carrotsearch.randomizedtesting.ThreadLeakControl$2 evaluate
   [junit4]   2> WARNING: Suite execution timed out: org.apache.lucene.facet.taxonomy.TestSearcherTaxonomyManager
   [junit4]   2>    1) Thread[id=88, name=Thread-56, state=RUNNABLE, group=TGRP-TestSearcherTaxonomyManager]
   [junit4]   2>         at org.apache.lucene.util.fst.FST.readNextRealArc(FST.java:1099)
   [junit4]   2>         at org.apache.lucene.util.fst.FST.readFirstRealTargetArc(FST.java:1001)
   [junit4]   2>         at org.apache.lucene.util.fst.NodeHash.nodesEqual(NodeHash.java:43)
   [junit4]   2>         at org.apache.lucene.util.fst.NodeHash.add(NodeHash.java:136)
   [junit4]   2>         at org.apache.lucene.util.fst.Builder.compileNode(Builder.java:189)
   [junit4]   2>         at org.apache.lucene.util.fst.Builder.freezeTail(Builder.java:277)
   [junit4]   2>         at org.apache.lucene.util.fst.Builder.add(Builder.java:381)
   [junit4]   2>         at org.apache.lucene.codecs.memory.FSTOrdTermsWriter$TermsWriter.finishTerm(FSTOrdTermsWriter.java:349)
   [junit4]   2>         at org.apache.lucene.codecs.memory.FSTOrdTermsWriter.write(FSTOrdTermsWriter.java:210)
   [junit4]   2>         at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:198)
   [junit4]   2>         at org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:193)
   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:95)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3944)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3525)
   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:1814)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4583)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:704)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4609)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4600)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1361)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1139)
   [junit4]   2>         at org.apache.lucene.facet.taxonomy.TestSearcherTaxonomyManager$IndexerThread.run(TestSearcherTaxonomyManager.java:95)
   [junit4]   2>    2) Thread[id=1, name=main, state=WAITING, group=main]
   [junit4]   2>         at java.lang.Object.wait(Native Method)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1281)
   [junit4]   2>         at java.lang.Thread.join(Thread.java:1355)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:578)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.run(RandomizedRunner.java:444)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.execute(SlaveMain.java:199)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMain.main(SlaveMain.java:310)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.slave.SlaveMainSafe.main(SlaveMainSafe.java:12)
   [junit4]   2>    3) Thread[id=9, name=JUnit4-serializer-daemon, state=TIMED_WAITING, group=main]
   [junit4]   2>         at java.lang.Thread.sleep(Native Method)
   [junit4]   2>         at com.carrotsearch.ant.tasks.junit4.events.Serializer$1.run(Serializer.java:47)
   [junit4]   2>    4) Thread[id=87, name=TEST-TestSearcherTaxonomyManager.testDirectory-seed#[39FAAAC972D62F74], state=RUNNABLE, group=TGRP-TestSearcherTaxonomyManager]
   [junit4]   2>         at org.apache.lucene.store.RAMInputStream.readBytes(RAMInputStream.java:90)
   [junit4]   2>         at org.apache.lucene.store.MockIndexInputWrapper.readBytes(MockIndexInputWrapper.java:138)
   [junit4]   2>         at org.apache.lucene.codecs.lucene50.Lucene50DocValuesProducer$6.get(Lucene50DocValuesProducer.java:519)
   [junit4]   2>         at org.apache.lucene.codecs.lucene50.Lucene50DocValuesProducer$LongBinaryDocValues.get(Lucene50DocValuesProducer.java:858)
   [junit4]   2>         at org.apache.lucene.facet.taxonomy.DocValuesOrdinalsReader$1.get(DocValuesOrdinalsReader.java:57)
   [junit4]   2>         at org.apache.lucene.facet.taxonomy.CachedOrdinalsReader$CachedOrds.<init>(CachedOrdinalsReader.java:122)
   [junit4]   2>         at org.apache.lucene.facet.taxonomy.CachedOrdinalsReader.getCachedOrds(CachedOrdinalsReader.java:75)
   [junit4]   2>         at org.apache.lucene.facet.taxonomy.CachedOrdinalsReader.getReader(CachedOrdinalsReader.java:89)
   [junit4]   2>         at org.apache.lucene.facet.taxonomy.TaxonomyFacetCounts.count(TaxonomyFacetCounts.java:51)
   [junit4]   2>         at org.apache.lucene.facet.taxonomy.TaxonomyFacetCounts.<init>(TaxonomyFacetCounts.java:45)
   [junit4]   2>         at org.apache.lucene.facet.FacetTestCase.getTaxonomyFacetCounts(FacetTestCase.java:54)
   [junit4]   2>         at org.apache.lucene.facet.FacetTestCase.getTaxonomyFacetCounts(FacetTestCase.java:42)
   [junit4]   2>         at org.apache.lucene.facet.taxonomy.TestSearcherTaxonomyManager.testDirectory(TestSearcherTaxonomyManager.java:236)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:606)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2>    5) Thread[id=86, name=SUITE-TestSearcherTaxonomyManager-seed#[39FAAAC972D62F74], state=RUNNABLE, group=TGRP-TestSearcherTaxonomyManager]
   [junit4]   2>         at java.lang.Thread.getStackTrace(Thread.java:1589)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.getThreadsWithTraces(ThreadLeakControl.java:690)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.formatThreadStacksFull(ThreadLeakControl.java:679)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.access$900(ThreadLeakControl.java:62)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$2.evaluate(ThreadLeakControl.java:412)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSuite(RandomizedRunner.java:651)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.access$200(RandomizedRunner.java:138)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$1.run(RandomizedRunner.java:568)
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSearcherTaxonomyManager -Dtests.method=testDirectory -Dtests.seed=39FAAAC972D62F74 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=be -Dtests.timezone=America/Managua -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   7200s J2 | TestSearcherTaxonomyManager.testDirectory <<<
   [junit4]    > Throwable #1: java.lang.Exception: Test abandoned because suite timeout was reached.
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([39FAAAC972D62F74]:0)
   [junit4]   2> ??? 09, 2015 11:12:48 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> WARNING: Will linger awaiting termination of 2 leaked thread(s).
   [junit4]   2> ??? 09, 2015 11:13:08 AM com.carrotsearch.randomizedtesting.ThreadLeakControl checkThreadLeaks
   [junit4]   2> SEVERE: 2 threads leaked from SUITE scope at org.apache.lucene.facet.taxonomy.TestSearcherTaxonomyManager: 
   [junit4]   2>    1) Thread[id=88, name=Thread-56, state=RUNNABLE, group=TGRP-TestSearcherTaxonomyManager]
   [junit4]   2>         at org.apache.lucene.util.fst.Builder.validOutput(Builder.java:440)
   [junit4]   2>         at org.apache.lucene.util.fst.Builder.access$100(Builder.java:53)
   [junit4]   2>         at org.apache.lucene.util.fst.Builder$UnCompiledNode.prependOutput(Builder.java:619)
   [junit4]   2>         at org.apache.lucene.util.fst.Builder.add(Builder.java:414)
   [junit4]   2>         at org.apache.lucene.codecs.memory.FSTOrdTermsWriter$TermsWriter.finishTerm(FSTOrdTermsWriter.java:349)
   [junit4]   2>         at org.apache.lucene.codecs.memory.FSTOrdTermsWriter.write(FSTOrdTermsWriter.java:210)
   [junit4]   2>         at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:198)
   [junit4]   2>         at org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:193)
   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:95)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3944)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3525)
   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:1814)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4583)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:704)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4609)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4600)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1361)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1139)
   [junit4]   2>         at org.apache.lucene.facet.taxonomy.TestSearcherTaxonomyManager$IndexerThread.run(TestSearcherTaxonomyManager.java:95)
   [junit4]   2>    2) Thread[id=87, name=TEST-TestSearcherTaxonomyManager.testDirectory-seed#[39FAAAC972D62F74], state=RUNNABLE, group=TGRP-TestSearcherTaxonomyManager]
   [junit4]   2>         at org.apache.lucene.util.packed.MonotonicBlockPackedReader.get(MonotonicBlockPackedReader.java:108)
   [junit4]   2>         at org.apache.lucene.codecs.lucene50.Lucene50DocValuesProducer$6.get(Lucene50DocValuesProducer.java:514)
   [junit4]   2>         at org.apache.lucene.codecs.lucene50.Lucene50DocValuesProducer$LongBinaryDocValues.get(Lucene50DocValuesProducer.java:858)
   [junit4]   2>         at org.apache.lucene.facet.taxonomy.FastTaxonomyFacetCounts.count(FastTaxonomyFacetCounts.java:63)
   [junit4]   2>         at org.apache.lucene.facet.taxonomy.FastTaxonomyFacetCounts.<init>(FastTaxonomyFacetCounts.java:49)
   [junit4]   2>         at org.apache.lucene.facet.FacetTestCase.getTaxonomyFacetCounts(FacetTestCase.java:48)
   [junit4]   2>         at org.apache.lucene.facet.FacetTestCase.getTaxonomyFacetCounts(FacetTestCase.java:42)
   [junit4]   2>         at org.apache.lucene.facet.taxonomy.TestSearcherTaxonomyManager.testDirectory(TestSearcherTaxonomyManager.java:236)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:606)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> ??? 09, 2015 11:13:08 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> INFO: Starting to interrupt leaked threads:
   [junit4]   2>    1) Thread[id=88, name=Thread-56, state=RUNNABLE, group=TGRP-TestSearcherTaxonomyManager]
   [junit4]   2>    2) Thread[id=87, name=TEST-TestSearcherTaxonomyManager.testDirectory-seed#[39FAAAC972D62F74], state=RUNNABLE, group=TGRP-TestSearcherTaxonomyManager]
   [junit4]   2> ??? 09, 2015 11:13:11 AM com.carrotsearch.randomizedtesting.ThreadLeakControl tryToInterruptAll
   [junit4]   2> SEVERE: There are still zombie threads that couldn't be terminated:
   [junit4]   2>    1) Thread[id=88, name=Thread-56, state=RUNNABLE, group=TGRP-TestSearcherTaxonomyManager]
   [junit4]   2>         at org.apache.lucene.util.fst.FSTEnum.pushFirst(FSTEnum.java:488)
   [junit4]   2>         at org.apache.lucene.util.fst.FSTEnum.doNext(FSTEnum.java:111)
   [junit4]   2>         at org.apache.lucene.util.fst.BytesRefFSTEnum.next(BytesRefFSTEnum.java:57)
   [junit4]   2>         at org.apache.lucene.codecs.memory.FSTOrdTermsReader$TermsReader$SegmentTermsEnum.next(FSTOrdTermsReader.java:499)
   [junit4]   2>         at org.apache.lucene.index.MultiTermsEnum.pushTop(MultiTermsEnum.java:269)
   [junit4]   2>         at org.apache.lucene.index.MultiTermsEnum.next(MultiTermsEnum.java:295)
   [junit4]   2>         at org.apache.lucene.index.FilterLeafReader$FilterTermsEnum.next(FilterLeafReader.java:194)
   [junit4]   2>         at org.apache.lucene.codecs.memory.FSTOrdTermsWriter.write(FSTOrdTermsWriter.java:204)
   [junit4]   2>         at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:198)
   [junit4]   2>         at org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:193)
   [junit4]   2>         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:95)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3944)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3525)
   [junit4]   2>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:1814)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4583)
   [junit4]   2>         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:704)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4609)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4600)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1361)
   [junit4]   2>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1139)
   [junit4]   2>         at org.apache.lucene.facet.taxonomy.TestSearcherTaxonomyManager$IndexerThread.run(TestSearcherTaxonomyManager.java:95)
   [junit4]   2>    2) Thread[id=87, name=TEST-TestSearcherTaxonomyManager.testDirectory-seed#[39FAAAC972D62F74], state=RUNNABLE, group=TGRP-TestSearcherTaxonomyManager]
   [junit4]   2>         at org.apache.lucene.store.RAMInputStream.readBytes(RAMInputStream.java:90)
   [junit4]   2>         at org.apache.lucene.store.MockIndexInputWrapper.readBytes(MockIndexInputWrapper.java:138)
   [junit4]   2>         at org.apache.lucene.codecs.lucene50.Lucene50DocValuesProducer$6.get(Lucene50DocValuesProducer.java:519)
   [junit4]   2>         at org.apache.lucene.codecs.lucene50.Lucene50DocValuesProducer$LongBinaryDocValues.get(Lucene50DocValuesProducer.java:858)
   [junit4]   2>         at org.apache.lucene.facet.taxonomy.DocValuesOrdinalsReader$1.get(DocValuesOrdinalsReader.java:57)
   [junit4]   2>         at org.apache.lucene.facet.taxonomy.TaxonomyFacetCounts.count(TaxonomyFacetCounts.java:56)
   [junit4]   2>         at org.apache.lucene.facet.taxonomy.TaxonomyFacetCounts.<init>(TaxonomyFacetCounts.java:45)
   [junit4]   2>         at org.apache.lucene.facet.FacetTestCase.getTaxonomyFacetCounts(FacetTestCase.java:54)
   [junit4]   2>         at org.apache.lucene.facet.FacetTestCase.getTaxonomyFacetCounts(FacetTestCase.java:42)
   [junit4]   2>         at org.apache.lucene.facet.taxonomy.TestSearcherTaxonomyManager.testDirectory(TestSearcherTaxonomyManager.java:236)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
   [junit4]   2>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2>         at java.lang.reflect.Method.invoke(Method.java:606)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
   [junit4]   2>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
   [junit4]   2>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
   [junit4]   2>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
   [junit4]   2>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
   [junit4]   2>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
   [junit4]   2>         at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {$facets=FSTOrd50, $payloads$=FST50, $full_path$=FST50}, docValues:{$facets=DocValuesFormat(name=Lucene50)}, sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=be, timezone=America/Managua
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=3,free=192153976,total=492830720
   [junit4]   2> NOTE: All tests run in this JVM: [TestCachedOrdinalsReader, TestTaxonomyFacetSumValueSource, TestTaxonomyFacetCounts, TestDirectoryTaxonomyWriter, TestConcurrentFacetedIndexing, TestDirectoryTaxonomyReader, TestFacetLabel, TestLRUHashMap, TestOrdinalMappingLeafReader, TestSearcherTaxonomyManager]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestSearcherTaxonomyManager -Dtests.seed=39FAAAC972D62F74 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=be -Dtests.timezone=America/Managua -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J2 | TestSearcherTaxonomyManager (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([39FAAAC972D62F74]:0)
   [junit4] Completed [23/23] on J2 in 7223.29s, 1 test, 2 errors <<< FAILURES!

[...truncated 13 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:543: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:484: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:457: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:2230: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/module-build.xml:58: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1434: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:991: There were test failures: 23 suites, 152 tests, 1 suite-level error, 1 error

Total time: 219 minutes 10 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 238480 bytes
Compression is 0.0%
Took 0.22 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 811 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/811/

1 tests failed.
REGRESSION:  org.apache.solr.handler.component.DistributedMLTComponentTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:51188//collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:51188//collection1
	at __randomizedtesting.SeedInfo.seed([18D4ED128EC399FC:9080D2C8203FF404]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:566)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:558)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:606)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:588)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:567)
	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 51 more




Build Log:
[...truncated 10633 lines...]
   [junit4] Suite: org.apache.solr.handler.component.DistributedMLTComponentTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/init-core-data-001
   [junit4]   2> 1296647 T8038 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 1296648 T8038 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1297831 T8038 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/control/cores/collection1
   [junit4]   2> 1297833 T8038 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1297837 T8038 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:51187
   [junit4]   2> 1297837 T8038 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=51187, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/control/cores}
   [junit4]   2> 1297837 T8038 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 1297838 T8038 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/control/'
   [junit4]   2> 1297868 T8038 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/control/solr.xml
   [junit4]   2> 1297906 T8038 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/control/cores
   [junit4]   2> 1297906 T8038 oasc.CoreContainer.<init> New CoreContainer 1863366120
   [junit4]   2> 1297907 T8038 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/control/]
   [junit4]   2> 1297907 T8038 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/control/lib
   [junit4]   2> 1297907 T8038 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/control/lib).
   [junit4]   2> 1297916 T8038 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1297916 T8038 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1297917 T8038 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1297917 T8038 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1297917 T8038 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1297917 T8038 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1297917 T8038 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1297918 T8038 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1297918 T8038 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1297918 T8038 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1297918 T8038 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1297924 T8038 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1297924 T8038 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1297925 T8038 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1297925 T8038 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1297926 T8038 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/control/cores
   [junit4]   2> 1297927 T8038 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/control/cores/collection1/, schema=schema.xml, loadOnStartup=true, configSet=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/control/cores/collection1, transient=false, config=solrconfig.xml, dataDir=data/, name=collection1}
   [junit4]   2> 1297928 T8038 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/control/cores/collection1/
   [junit4]   2> 1297928 T8038 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1297929 T8048 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 1297932 T8048 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 1297932 T8048 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 1297933 T8048 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 1297972 T8048 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1297993 T8048 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1298044 T8048 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1298045 T8048 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 1298052 T8048 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1298347 T8048 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1298359 T8048 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1298361 T8048 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1298367 T8048 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1298371 T8048 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1298373 T8048 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1298374 T8048 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1298374 T8048 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1298374 T8048 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1298374 T8048 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1298375 T8048 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1298375 T8048 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from configset /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1
   [junit4]   2> 1298375 T8048 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1298375 T8048 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/control/cores/collection1/data/
   [junit4]   2> 1298376 T8048 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@52b75145
   [junit4]   2> 1298376 T8048 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/control/cores/collection1/data
   [junit4]   2> 1298376 T8048 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/control/cores/collection1/data/index/
   [junit4]   2> 1298376 T8048 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/control/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1298377 T8048 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/control/cores/collection1/data/index
   [junit4]   2> 1298377 T8048 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=48, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=1.7998046875, noCFSRatio=1.0]
   [junit4]   2> 1298378 T8048 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1b7175f0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@35f5eda3),segFN=segments_1,generation=1}
   [junit4]   2> 1298378 T8048 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1298381 T8048 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1298382 T8048 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1298382 T8048 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 1298382 T8048 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 1298382 T8048 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1298382 T8048 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1298382 T8048 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 1298383 T8048 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 1298383 T8048 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1298383 T8048 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1298383 T8048 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1298383 T8048 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1298397 T8048 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1298398 T8048 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1298400 T8048 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1298401 T8048 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1298405 T8048 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 1298407 T8048 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/admin/file,/admin/system,/mlt,/schema,spellCheckWithWordbreak_Direct,standard,/search-facet-def,/admin/plugins,/admin/threads,dismax,/debug/dump,/admin/ping,spellCheckWithWordbreak,/admin/luke,tvrh,lazy,/admin/mbeans,/get,/search-facet-invariants,/admin/segments,/terms,/update/csv,/replication,mock,/admin/logging,spellCheckCompRH1,/config,/update/json/docs,defaults,/admin/properties,mltrh,spellCheckCompRH_Direct,/update/json,spellCheckCompRH
   [junit4]   2> 1298408 T8048 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 1298409 T8048 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1298410 T8048 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1298410 T8048 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1298411 T8048 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1298412 T8048 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1298412 T8048 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@5e5f2ac4
   [junit4]   2> 1298412 T8048 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1b7175f0 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@35f5eda3),segFN=segments_1,generation=1}
   [junit4]   2> 1298413 T8048 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1298413 T8048 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@417acf13[collection1] main
   [junit4]   2> 1298413 T8048 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 1298413 T8048 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 1298414 T8048 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1298414 T8048 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 1298448 T8048 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 1298507 T8048 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1298507 T8049 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 1298508 T8049 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 1298508 T8049 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 1298508 T8049 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 1298509 T8048 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1298509 T8038 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 1298510 T8038 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1298509 T8049 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 1298511 T8049 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 1298511 T8049 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 1298512 T8049 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 1298513 T8049 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 1298514 T8049 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@417acf13[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1299942 T8038 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard0/cores/collection1
   [junit4]   2> 1299945 T8038 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1299949 T8038 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:51188
   [junit4]   2> 1299949 T8038 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, hostPort=51188, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard0/cores}
   [junit4]   2> 1299950 T8038 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 1299950 T8038 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard0/'
   [junit4]   2> 1299991 T8038 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard0/solr.xml
   [junit4]   2> 1300072 T8038 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard0/cores
   [junit4]   2> 1300073 T8038 oasc.CoreContainer.<init> New CoreContainer 854900077
   [junit4]   2> 1300073 T8038 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard0/]
   [junit4]   2> 1300074 T8038 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard0/lib
   [junit4]   2> 1300075 T8038 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard0/lib).
   [junit4]   2> 1300089 T8038 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1300090 T8038 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1300090 T8038 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1300090 T8038 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1300090 T8038 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1300090 T8038 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1300091 T8038 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1300091 T8038 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1300091 T8038 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1300091 T8038 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1300091 T8038 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1300098 T8038 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1300099 T8038 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1300099 T8038 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1300099 T8038 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1300100 T8038 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard0/cores
   [junit4]   2> 1300102 T8038 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard0/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard0/cores/collection1/, configSet=collection1, transient=false, dataDir=data/, schema=schema.xml, config=solrconfig.xml}
   [junit4]   2> 1300102 T8038 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard0/cores/collection1/
   [junit4]   2> 1300103 T8038 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1300103 T8060 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 1300108 T8060 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 1300108 T8060 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 1300109 T8060 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 1300155 T8060 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1300177 T8060 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1300230 T8060 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1300231 T8060 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 1300238 T8060 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1300519 T8060 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1300531 T8060 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1300533 T8060 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1300539 T8060 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1300542 T8060 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1300545 T8060 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1300546 T8060 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1300546 T8060 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1300546 T8060 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1300546 T8060 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1300547 T8060 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1300547 T8060 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from configset /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1
   [junit4]   2> 1300547 T8060 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1300547 T8060 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard0/cores/collection1/data/
   [junit4]   2> 1300547 T8060 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@52b75145
   [junit4]   2> 1300548 T8060 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard0/cores/collection1/data
   [junit4]   2> 1300548 T8060 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard0/cores/collection1/data/index/
   [junit4]   2> 1300548 T8060 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard0/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1300549 T8060 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard0/cores/collection1/data/index
   [junit4]   2> 1300549 T8060 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=48, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=1.7998046875, noCFSRatio=1.0]
   [junit4]   2> 1300549 T8060 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@40bdbee9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6940d433),segFN=segments_1,generation=1}
   [junit4]   2> 1300550 T8060 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1300553 T8060 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1300554 T8060 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1300554 T8060 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 1300554 T8060 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 1300554 T8060 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1300554 T8060 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1300555 T8060 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 1300555 T8060 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 1300555 T8060 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1300555 T8060 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1300555 T8060 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1300556 T8060 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1300570 T8060 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1300572 T8060 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1300573 T8060 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1300575 T8060 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1300579 T8060 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 1300582 T8060 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/admin/file,/admin/system,/mlt,/schema,spellCheckWithWordbreak_Direct,standard,/search-facet-def,/admin/plugins,/admin/threads,dismax,/debug/dump,/admin/ping,spellCheckWithWordbreak,/admin/luke,tvrh,lazy,/admin/mbeans,/get,/search-facet-invariants,/admin/segments,/terms,/update/csv,/replication,mock,/admin/logging,spellCheckCompRH1,/config,/update/json/docs,defaults,/admin/properties,mltrh,spellCheckCompRH_Direct,/update/json,spellCheckCompRH
   [junit4]   2> 1300583 T8060 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 1300584 T8060 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1300585 T8060 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1300585 T8060 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1300586 T8060 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1300586 T8060 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1300587 T8060 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@1c297d80
   [junit4]   2> 1300587 T8060 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@40bdbee9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6940d433),segFN=segments_1,generation=1}
   [junit4]   2> 1300587 T8060 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1300588 T8060 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@591b3936[collection1] main
   [junit4]   2> 1300588 T8060 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 1300588 T8060 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 1300588 T8060 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1300588 T8060 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 1300627 T8060 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 1300697 T8060 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1300697 T8061 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 1300698 T8061 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 1300698 T8061 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 1300698 T8060 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1300698 T8061 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 1300698 T8038 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 1300699 T8038 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1300699 T8061 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 1300700 T8061 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 1300700 T8061 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 1300701 T8061 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 1300701 T8061 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 1300702 T8061 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@591b3936[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1301787 T8038 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard1/cores/collection1
   [junit4]   2> 1301790 T8038 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1301794 T8038 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:51189
   [junit4]   2> 1301794 T8038 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard1/cores, hostContext=/, hostPort=51189}
   [junit4]   2> 1301795 T8038 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 1301795 T8038 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard1/'
   [junit4]   2> 1301830 T8038 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard1/solr.xml
   [junit4]   2> 1301871 T8038 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard1/cores
   [junit4]   2> 1301871 T8038 oasc.CoreContainer.<init> New CoreContainer 196666758
   [junit4]   2> 1301872 T8038 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard1/]
   [junit4]   2> 1301872 T8038 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard1/lib
   [junit4]   2> 1301872 T8038 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard1/lib).
   [junit4]   2> 1301883 T8038 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1301884 T8038 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1301884 T8038 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1301884 T8038 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1301885 T8038 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1301885 T8038 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1301885 T8038 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1301885 T8038 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1301885 T8038 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1301886 T8038 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1301886 T8038 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1301891 T8038 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1301892 T8038 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1301892 T8038 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1301893 T8038 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1301893 T8038 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard1/cores
   [junit4]   2> 1301895 T8038 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard1/cores/collection1, transient=false, loadOnStartup=true, dataDir=data/, schema=schema.xml, name=collection1, configSet=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard1/cores/collection1/}
   [junit4]   2> 1301895 T8038 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard1/cores/collection1/
   [junit4]   2> 1301895 T8038 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1301896 T8072 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 1301900 T8072 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 1301900 T8072 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 1301901 T8072 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 1301951 T8072 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1301980 T8072 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1302045 T8072 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1302046 T8072 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 1302055 T8072 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1302352 T8072 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1302364 T8072 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1302367 T8072 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1302374 T8072 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1302392 T8072 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1302396 T8072 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1302397 T8072 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1302397 T8072 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1302397 T8072 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1302398 T8072 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1302398 T8072 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1302398 T8072 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from configset /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1
   [junit4]   2> 1302398 T8072 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1302399 T8072 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard1/cores/collection1/data/
   [junit4]   2> 1302399 T8072 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@52b75145
   [junit4]   2> 1302402 T8072 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard1/cores/collection1/data
   [junit4]   2> 1302402 T8072 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard1/cores/collection1/data/index/
   [junit4]   2> 1302402 T8072 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard1/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1302403 T8072 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard1/cores/collection1/data/index
   [junit4]   2> 1302403 T8072 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=48, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=1.7998046875, noCFSRatio=1.0]
   [junit4]   2> 1302404 T8072 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@30de634d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@58496d4b),segFN=segments_1,generation=1}
   [junit4]   2> 1302404 T8072 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1302409 T8072 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1302410 T8072 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1302410 T8072 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 1302410 T8072 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 1302410 T8072 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1302411 T8072 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1302411 T8072 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 1302411 T8072 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 1302411 T8072 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1302412 T8072 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1302412 T8072 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1302412 T8072 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1302433 T8072 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1302435 T8072 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1302437 T8072 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1302439 T8072 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1302445 T8072 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 1302448 T8072 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/admin/file,/admin/system,/mlt,/schema,spellCheckWithWordbreak_Direct,standard,/search-facet-def,/admin/plugins,/admin/threads,dismax,/debug/dump,/admin/ping,spellCheckWithWordbreak,/admin/luke,tvrh,lazy,/admin/mbeans,/get,/search-facet-invariants,/admin/segments,/terms,/update/csv,/replication,mock,/admin/logging,spellCheckCompRH1,/config,/update/json/docs,defaults,/admin/properties,mltrh,spellCheckCompRH_Direct,/update/json,spellCheckCompRH
   [junit4]   2> 1302450 T8072 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 1302451 T8072 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1302453 T8072 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1302453 T8072 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1302454 T8072 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1302455 T8072 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1302455 T8072 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@17ab67bc
   [junit4]   2> 1302456 T8072 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@30de634d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@58496d4b),segFN=segments_1,generation=1}
   [junit4]   2> 1302456 T8072 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1302456 T8072 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@65c2fa79[collection1] main
   [junit4]   2> 1302457 T8072 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 1302457 T8072 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 1302457 T8072 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1302457 T8072 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 1302494 T8072 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 1302540 T8072 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1302540 T8073 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 1302541 T8073 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 1302541 T8073 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 1302541 T8072 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1302541 T8073 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 1302542 T8038 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 1302543 T8073 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 1302543 T8038 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1302543 T8073 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 1302544 T8073 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 1302545 T8073 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 1302546 T8073 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 1302547 T8073 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@65c2fa79[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1303647 T8038 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard2/cores/collection1
   [junit4]   2> 1303649 T8038 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1303654 T8038 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:51192
   [junit4]   2> 1303654 T8038 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard2/cores, hostContext=/, hostPort=51192}
   [junit4]   2> 1303654 T8038 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 1303654 T8038 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard2/'
   [junit4]   2> 1303690 T8038 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard2/solr.xml
   [junit4]   2> 1303730 T8038 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard2/cores
   [junit4]   2> 1303730 T8038 oasc.CoreContainer.<init> New CoreContainer 1353870059
   [junit4]   2> 1303731 T8038 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard2/]
   [junit4]   2> 1303731 T8038 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard2/lib
   [junit4]   2> 1303731 T8038 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard2/lib).
   [junit4]   2> 1303742 T8038 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1303742 T8038 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1303743 T8038 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1303743 T8038 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1303743 T8038 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1303743 T8038 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1303743 T8038 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1303744 T8038 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1303744 T8038 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1303744 T8038 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1303744 T8038 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1303749 T8038 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1303750 T8038 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1303750 T8038 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1303751 T8038 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1303752 T8038 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard2/cores
   [junit4]   2> 1303753 T8038 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, configSet=collection1, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard2/cores/collection1/, name=collection1, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard2/cores/collection1, loadOnStartup=true, transient=false}
   [junit4]   2> 1303754 T8038 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard2/cores/collection1/
   [junit4]   2> 1303754 T8038 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1303755 T8084 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 1303758 T8084 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 1303759 T8084 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 1303759 T8084 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 1303811 T8084 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1303843 T8084 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1303913 T8084 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1303914 T8084 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 1303924 T8084 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1304329 T8084 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1304345 T8084 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1304348 T8084 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1304357 T8084 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1304361 T8084 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1304365 T8084 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1304366 T8084 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1304366 T8084 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1304366 T8084 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1304367 T8084 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1304367 T8084 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1304367 T8084 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from configset /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1
   [junit4]   2> 1304367 T8084 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1304368 T8084 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard2/cores/collection1/data/
   [junit4]   2> 1304368 T8084 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@52b75145
   [junit4]   2> 1304369 T8084 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard2/cores/collection1/data
   [junit4]   2> 1304369 T8084 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard2/cores/collection1/data/index/
   [junit4]   2> 1304369 T8084 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard2/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1304370 T8084 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard2/cores/collection1/data/index
   [junit4]   2> 1304370 T8084 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=48, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=1.7998046875, noCFSRatio=1.0]
   [junit4]   2> 1304371 T8084 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7b478ee7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@16f71eec),segFN=segments_1,generation=1}
   [junit4]   2> 1304371 T8084 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1304377 T8084 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1304377 T8084 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1304377 T8084 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 1304378 T8084 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 1304378 T8084 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1304378 T8084 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1304378 T8084 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 1304379 T8084 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 1304379 T8084 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1304379 T8084 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1304380 T8084 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1304380 T8084 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1304398 T8084 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1304401 T8084 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1304403 T8084 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1304405 T8084 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1304412 T8084 c:collection1 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 1304414 T8084 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/admin/file,/admin/system,/mlt,/schema,spellCheckWithWordbreak_Direct,standard,/search-facet-def,/admin/plugins,/admin/threads,dismax,/debug/dump,/admin/ping,spellCheckWithWordbreak,/admin/luke,tvrh,lazy,/admin/mbeans,/get,/search-facet-invariants,/admin/segments,/terms,/update/csv,/replication,mock,/admin/logging,spellCheckCompRH1,/config,/update/json/docs,defaults,/admin/properties,mltrh,spellCheckCompRH_Direct,/update/json,spellCheckCompRH
   [junit4]   2> 1304416 T8084 c:collection1 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 1304417 T8084 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1304418 T8084 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1304418 T8084 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1304419 T8084 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1304419 T8084 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1304420 T8084 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@9547356
   [junit4]   2> 1304420 T8084 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7b478ee7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@16f71eec),segFN=segments_1,generation=1}
   [junit4]   2> 1304421 T8084 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1304421 T8084 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@9586aad[collection1] main
   [junit4]   2> 1304421 T8084 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 1304421 T8084 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 1304421 T8084 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1304422 T8084 c:collection1 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 1304438 T8084 c:collection1 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 1304491 T8084 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1304491 T8085 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 1304492 T8085 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 1304492 T8085 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 1304492 T8084 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1304492 T8085 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 1304493 T8038 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 1304493 T8085 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 1304493 T8038 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1304494 T8085 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 1304494 T8085 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 1304494 T8038 oas.SolrTestCaseJ4.setUp ###Starting test
   [junit4]   2> 1304495 T8085 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 1304495 T8085 c:collection1 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 1304499 T8085 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@9586aad[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> ASYNC  NEW_CORE C2443 name=collection1 org.apache.solr.core.SolrCore@5bd920b
   [junit4]   2> 1304500 T8041 c:collection1 C2443 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {deleteByQuery=*:* (-1497901217960951808)} 0 4
   [junit4]   2> ASYNC  NEW_CORE C2444 name=collection1 org.apache.solr.core.SolrCore@70798f1a
   [junit4]   2> 1304507 T8058 c:collection1 C2444 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {deleteByQuery=*:* (-1497901217968291840)} 0 5
   [junit4]   2> ASYNC  NEW_CORE C2445 name=collection1 org.apache.solr.core.SolrCore@7ac93f7f
   [junit4]   2> 1304512 T8066 c:collection1 C2445 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {deleteByQuery=*:* (-1497901217974583296)} 0 4
   [junit4]   2> ASYNC  NEW_CORE C2446 name=collection1 org.apache.solr.core.SolrCore@2616eebe
   [junit4]   2> 1304518 T8078 c:collection1 C2446 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {deleteByQuery=*:* (-1497901217980874752)} 0 4
   [junit4]   2> 1304523 T8041 c:collection1 C2443 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[1 (1497901217987166208)]} 0 2
   [junit4]   2> 1304526 T8058 c:collection1 C2444 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[1 (1497901217991360512)]} 0 1
   [junit4]   2> 1304529 T8041 c:collection1 C2443 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[2 (1497901217995554816)]} 0 0
   [junit4]   2> 1304533 T8066 c:collection1 C2445 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[2 (1497901217998700544)]} 0 1
   [junit4]   2> 1304536 T8041 c:collection1 C2443 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[3 (1497901218002894848)]} 0 0
   [junit4]   2> 1304540 T8078 c:collection1 C2446 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[3 (1497901218006040576)]} 0 1
   [junit4]   2> 1304543 T8041 c:collection1 C2443 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[4 (1497901218010234880)]} 0 0
   [junit4]   2> 1304546 T8058 c:collection1 C2444 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[4 (1497901218013380608)]} 0 0
   [junit4]   2> 1304549 T8041 c:collection1 C2443 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[5 (1497901218016526336)]} 0 0
   [junit4]   2> 1304552 T8066 c:collection1 C2445 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[5 (1497901218019672064)]} 0 0
   [junit4]   2> 1304555 T8041 c:collection1 C2443 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[6 (1497901218022817792)]} 0 0
   [junit4]   2> 1304558 T8078 c:collection1 C2446 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[6 (1497901218025963520)]} 0 0
   [junit4]   2> 1304562 T8041 c:collection1 C2443 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[7 (1497901218029109248)]} 0 1
   [junit4]   2> 1304565 T8058 c:collection1 C2444 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[7 (1497901218032254976)]} 0 1
   [junit4]   2> 1304567 T8041 c:collection1 C2443 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[8 (1497901218035400704)]} 0 0
   [junit4]   2> 1304570 T8066 c:collection1 C2445 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[8 (1497901218038546432)]} 0 0
   [junit4]   2> 1304573 T8041 c:collection1 C2443 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[9 (1497901218041692160)]} 0 0
   [junit4]   2> 1304576 T8078 c:collection1 C2446 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[9 (1497901218044837888)]} 0 0
   [junit4]   2> 1304579 T8041 c:collection1 C2443 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[10 (1497901218047983616)]} 0 0
   [junit4]   2> 1304581 T8058 c:collection1 C2444 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[10 (1497901218050080768)]} 0 0
   [junit4]   2> 1304584 T8041 c:collection1 C2443 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[12 (1497901218053226496)]} 0 0
   [junit4]   2> 1304587 T8078 c:collection1 C2446 oasup.LogUpdateProcessor.finish [collection1] webapp= path=/update params={wt=javabin&version=2} {add=[12 (1497901218056372224)]} 0 0
   [junit4]   2> 1304590 T8041 c:collection1 C2443 oasup.LogUpdateProcessor.finish [collection1] webapp= path

[...truncated too long message...]

rkspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard2/cores/collection1/data
   [junit4]   2> 1395814 T8038 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard2/cores/collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard2/cores/collection1/data/index;done=false>>]
   [junit4]   2> 1395814 T8038 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001/tempDir-001/shard2/cores/collection1/data/index
   [junit4]   2> 1395815 T8038 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=DistributedMLTComponentTest -Dtests.method=test -Dtests.seed=18D4ED128EC399FC -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=uk_UA -Dtests.timezone=Etc/GMT-2 -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   99.4s J3 | DistributedMLTComponentTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:51188//collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([18D4ED128EC399FC:9080D2C8203FF404]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:566)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:558)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:606)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:588)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:567)
   [junit4]    > 	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: java.net.SocketTimeoutException: Read timed out
   [junit4]    > 	at java.net.SocketInputStream.socketRead0(Native Method)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:152)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:122)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
   [junit4]    > 	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 51 more
   [junit4]   2> 1396060 T8038 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.handler.component.DistributedMLTComponentTest 18D4ED128EC399FC-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {intDefault=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), multiDefault=PostingsFormat(name=MockRandom), lowerfilt=PostingsFormat(name=MockRandom), id=PostingsFormat(name=LuceneVarGapDocFreqInterval), range_facet_l=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), timestamp=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), lowerfilt1and2=PostingsFormat(name=MockRandom), _version_=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), lowerfilt1=Lucene50(blocksize=128)}, docValues:{timestamp=DocValuesFormat(name=Asserting)}, sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=uk_UA, timezone=Etc/GMT-2
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=171423592,total=523239424
   [junit4]   2> NOTE: All tests run in this JVM: [TestUtils, DateMathParserTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, UpdateRequestProcessorFactoryTest, OutputWriterTest, TestLMDirichletSimilarityFactory, TestDocBasedVersionConstraints, DocValuesMissingTest, TestCloudPivotFacet, TestJmxIntegration, CursorMarkTest, DistributedSuggestComponentTest, CursorPagingTest, TestLRUStatsCache, CurrencyFieldXmlFileTest, UnloadDistributedZkTest, TestShardHandlerFactory, TestDistributedGrouping, SpellingQueryConverterTest, TestSortingResponseWriter, TestWordDelimiterFilterFactory, TestBulkSchemaAPI, TestHighlightDedupGrouping, TestRangeQuery, TestPostingsSolrHighlighter, TermVectorComponentDistributedTest, AliasIntegrationTest, CustomCollectionTest, RecoveryAfterSoftCommitTest, TestCodecSupport, TestReplicaProperties, BadIndexSchemaTest, DistribDocExpirationUpdateProcessorTest, ResponseHeaderTest, TestIndexSearcher, DefaultValueUpdateProcessorTest, MultiTermTest, CollectionsAPIDistributedZkTest, DistributedMLTComponentTest]
   [junit4] Completed on J3 in 99.45s, 1 test, 1 error <<< FAILURES!

[...truncated 855 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 480 suites, 1905 tests, 1 error, 58 ignored (9 assumptions)

Total time: 191 minutes 33 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 831140109 bytes
Compression is 0.0%
Took 2 min 36 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 810 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/810/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:51102/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51102/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([7764FDC9C7D71437:FF30C213692B79CF]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:51102 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19802/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19802/collection1
	at __randomizedtesting.SeedInfo.seed([7764FDC9C7D71437:FF30C213692B79CF]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19802/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:19802 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19834/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19834/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([7764FDC9C7D71437:FF30C213692B79CF]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19834/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:19834 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 10589 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/init-core-data-001
   [junit4]   2> 1851995 T10326 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 1852007 T10326 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1852008 T10327 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1852108 T10326 oasc.ZkTestServer.run start zk server on port:19776
   [junit4]   2> 1852109 T10326 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1852110 T10326 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1852114 T10334 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5c948ae6 name:ZooKeeperConnection Watcher:127.0.0.1:19776 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1852115 T10326 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1852115 T10326 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1852116 T10326 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1852120 T10326 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1852122 T10326 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1852124 T10337 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@34421e60 name:ZooKeeperConnection Watcher:127.0.0.1:19776/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1852124 T10326 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1852125 T10326 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1852125 T10326 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1852129 T10326 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1852133 T10326 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1852136 T10326 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1852141 T10326 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1852141 T10326 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1852145 T10326 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1852146 T10326 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1852250 T10326 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1852252 T10326 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1852255 T10326 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1852257 T10326 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1852261 T10326 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1852262 T10326 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1852266 T10326 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1852267 T10326 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1852270 T10326 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1852272 T10326 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1852277 T10326 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1852279 T10326 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1852283 T10326 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1852284 T10326 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1852291 T10326 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1852293 T10326 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1852296 T10326 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1852297 T10326 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1856174 T10326 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/control-001/cores/collection1
   [junit4]   2> 1856179 T10326 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1856182 T10326 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:19786
   [junit4]   2> 1856183 T10326 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/tempDir-001/control/data, hostContext=/, hostPort=19785}
   [junit4]   2> 1856183 T10326 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 1856183 T10326 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/control-001/'
   [junit4]   2> 1856382 T10326 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/control-001/solr.xml
   [junit4]   2> 1856520 T10326 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/control-001/cores
   [junit4]   2> 1856521 T10326 oasc.CoreContainer.<init> New CoreContainer 633139302
   [junit4]   2> 1856521 T10326 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/control-001/]
   [junit4]   2> 1856521 T10326 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/control-001/lib
   [junit4]   2> 1856523 T10326 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/control-001/lib).
   [junit4]   2> 1856547 T10326 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1856547 T10326 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1856548 T10326 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1856548 T10326 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1856549 T10326 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1856549 T10326 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1856550 T10326 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1856550 T10326 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1856551 T10326 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1856551 T10326 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1856551 T10326 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1856573 T10326 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1856576 T10326 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1856576 T10326 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1856577 T10326 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1856577 T10326 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:19776/solr
   [junit4]   2> 1856577 T10326 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1856578 T10326 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1856580 T10326 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1856582 T10349 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@535995e6 name:ZooKeeperConnection Watcher:127.0.0.1:19776 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1856583 T10326 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1856585 T10326 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1856588 T10326 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1856590 T10352 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@103bc680 name:ZooKeeperConnection Watcher:127.0.0.1:19776/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1856590 T10326 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1856593 T10326 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1856597 T10326 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1856600 T10326 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1856604 T10326 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1856607 T10326 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1856611 T10326 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1856613 T10326 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1856616 T10326 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1856618 T10326 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:19785_
   [junit4]   2> 1856619 T10326 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:19785_
   [junit4]   2> 1856623 T10326 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1856629 T10326 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1856630 T10326 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1856634 T10326 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:19785_
   [junit4]   2> 1856635 T10326 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1856637 T10326 oasc.Overseer.start Overseer (id=93613132110626819-127.0.0.1:19785_-n_0000000000) starting
   [junit4]   2> 1856645 T10326 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1856657 T10326 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1856658 T10354 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1856658 T10326 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1856659 T10353 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1856666 T10326 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/control-001/cores
   [junit4]   2> 1856671 T10326 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, loadOnStartup=true, collection=control_collection, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/control-001/cores/collection1, dataDir=data/, shard=, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/control-001/cores/collection1/, config=solrconfig.xml, coreNodeName=, schema=schema.xml}
   [junit4]   2> 1856672 T10326 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/control-001/cores/collection1/
   [junit4]   2> 1856672 T10326 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1856674 T10356 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1856674 T10356 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1856676 T10352 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1856676 T10356 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1856679 T10353 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:19785_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19785",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 0
   [junit4]   2> 1856683 T10353 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:19785_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19785",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 1856683 T10353 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1856684 T10353 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1856687 T10352 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1857678 T10356 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1857679 T10356 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1857681 T10356 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1857681 T10356 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1857685 T10356 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1857685 T10356 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/control-001/cores/collection1/'
   [junit4]   2> 1857722 T10356 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1857726 T10356 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1857744 T10356 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1857794 T10356 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1857907 T10356 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1858009 T10356 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1858036 T10356 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1858851 T10356 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1858961 T10356 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1858987 T10356 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1859013 T10356 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1859021 T10356 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1859028 T10356 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1859029 T10356 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1859030 T10356 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1859030 T10356 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1859031 T10356 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1859031 T10356 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1859032 T10356 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1859032 T10356 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1859033 T10356 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/control-001/cores/collection1/data/
   [junit4]   2> 1859033 T10356 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4eb85fa9
   [junit4]   2> 1859035 T10356 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/control-001/cores/collection1/data
   [junit4]   2> 1859035 T10356 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1859036 T10356 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1859036 T10356 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/control-001/cores/collection1/data/index
   [junit4]   2> 1859037 T10356 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=7, maxMergeAtOnceExplicit=4, maxMergedSegmentMB=23.8427734375, floorSegmentMB=0.431640625, forceMergeDeletesPctAllowed=10.29500517309118, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.10316662867532474
   [junit4]   2> 1859039 T10356 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@62ff153f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@58cc0ddf; maxCacheMB=0.8203125 maxMergeSizeMB=0.61328125)),segFN=segments_1,generation=1}
   [junit4]   2> 1859039 T10356 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1859048 T10356 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1859049 T10356 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1859050 T10356 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1859050 T10356 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1859050 T10356 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1859051 T10356 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1859052 T10356 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1859052 T10356 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1859053 T10356 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1859116 T10356 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1859120 T10356 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1859123 T10356 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1859128 T10356 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1859135 T10356 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/csv,/admin/segments,/schema,/admin/file,/admin/plugins,/config,/update/json/docs,/admin/luke,/admin/ping,/admin/logging,/get,/admin/properties,/replication,/admin/system,/update/json,/update,/admin/mbeans,/admin/threads,standard
   [junit4]   2> 1859139 T10356 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1859144 T10356 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1859144 T10356 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1859148 T10356 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1859149 T10356 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1859150 T10356 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@1f2eee43
   [junit4]   2> 1859151 T10356 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@62ff153f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@58cc0ddf; maxCacheMB=0.8203125 maxMergeSizeMB=0.61328125)),segFN=segments_1,generation=1}
   [junit4]   2> 1859151 T10356 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1859152 T10356 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@28a9a48[collection1] main
   [junit4]   2> 1859152 T10356 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1859154 T10356 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1859154 T10356 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1859155 T10356 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1859155 T10356 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1859155 T10356 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1859156 T10356 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1859157 T10356 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1859157 T10356 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1859158 T10356 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1859159 T10357 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@28a9a48[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1859160 T10356 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1859161 T10360 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:19785 collection:control_collection shard:shard1
   [junit4]   2> 1859161 T10326 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 1859162 T10326 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1859162 T10360 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1859168 T10326 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1859169 T10326 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1859170 T10364 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7fe465c2 name:ZooKeeperConnection Watcher:127.0.0.1:19776/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1859171 T10326 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1859171 T10326 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1859172 T10326 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1859172 T10360 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1859175 T10360 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1859175 T10360 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C933 name=collection1 org.apache.solr.core.SolrCore@d23f087 url=http://127.0.0.1:19785/collection1 node=127.0.0.1:19785_ C933_STATE=coll:control_collection core:collection1 props:{core=collection1, node_name=127.0.0.1:19785_, state=down, base_url=http://127.0.0.1:19785}
   [junit4]   2> 1859175 T10360 C:control_collection S:shard1 C933 P19785 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:19785/collection1/
   [junit4]   2> 1859175 T10326 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1859176 T10352 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1859176 T10326 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 1859176 T10360 C:control_collection S:shard1 C933 P19785 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1859177 T10326 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1859177 T10353 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1859177 T10360 C:control_collection S:shard1 C933 P19785 oasc.SyncStrategy.syncToMe http://127.0.0.1:19785/collection1/ has no replicas
   [junit4]   2> 1859178 T10326 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1859178 T10360 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:19785/collection1/ shard1
   [junit4]   2> 1859179 T10360 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1859181 T10352 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1859181 T10367 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@104d7f18 name:ZooKeeperConnection Watcher:127.0.0.1:19776/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1859181 T10364 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1859182 T10326 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1859184 T10326 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1859188 T10353 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 1859189 T10353 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1859190 T10352 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1859192 T10353 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19785",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1859193 T10353 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 1859195 T10352 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1859195 T10364 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1859201 T10352 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1859202 T10364 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1859202 T10352 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1859242 T10360 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:19785/collection1/ and leader is http://127.0.0.1:19785/collection1/
   [junit4]   2> 1859243 T10360 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:19785
   [junit4]   2> 1859243 T10360 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1859243 T10360 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1859244 T10360 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1859245 T10352 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1859246 T10353 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:19785_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19785",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 4
   [junit4]   2> 1859247 T10353 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:19785_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19785",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 1859351 T10364 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1859351 T10352 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1863240 T10326 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/shard-1-001/cores/collection1
   [junit4]   2> 1863245 T10326 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/shard-1-001
   [junit4]   2> 1863246 T10326 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1863248 T10326 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:19803
   [junit4]   2> 1863249 T10326 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, hostPort=19802, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/shard-1-001/cores, hostContext=/}
   [junit4]   2> 1863250 T10326 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 1863250 T10326 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/shard-1-001/'
   [junit4]   2> 1863397 T10326 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/shard-1-001/solr.xml
   [junit4]   2> 1863536 T10326 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/shard-1-001/cores
   [junit4]   2> 1863537 T10326 oasc.CoreContainer.<init> New CoreContainer 941002770
   [junit4]   2> 1863537 T10326 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/shard-1-001/]
   [junit4]   2> 1863538 T10326 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/shard-1-001/lib
   [junit4]   2> 1863540 T10326 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/shard-1-001/lib).
   [junit4]   2> 1863563 T10326 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1863564 T10326 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1863564 T10326 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1863565 T10326 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1863565 T10326 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1863566 T10326 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1863566 T10326 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1863567 T10326 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1863567 T10326 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1863568 T10326 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1863568 T10326 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1863598 T10326 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1863600 T10326 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1863601 T10326 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1863601 T10326 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1863602 T10326 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:19776/solr
   [junit4]   2> 1863602 T10326 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1863603 T10326 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1863604 T10326 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1863607 T10379 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@17dbc3b4 name:ZooKeeperConnection Watcher:127.0.0.1:19776 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1863608 T10326 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1863610 T10326 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1863613 T10326 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1863615 T10382 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7130e96e name:ZooKeeperConnection Watcher:127.0.0.1:19776/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1863615 T10326 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1863621 T10326 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1864632 T10326 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:19802_
   [junit4]   2> 1864634 T10326 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:19802_
   [junit4]   2> 1864640 T10326 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1864646 T10326 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/shard-1-001/cores
   [junit4]   2> 1864652 T10326 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/shard-1-001/cores/collection1/, transient=false, dataDir=data/, config=solrconfig.xml, loadOnStartup=true, shard=, coreNodeName=, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/shard-1-001/cores/collection1}
   [junit4]   2> 1864653 T10326 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/shard-1-001/cores/collection1/
   [junit4]   2> 1864653 T10326 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1864657 T10383 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1864658 T10383 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1864660 T10352 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1864661 T10353 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:19802_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19802",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 1864661 T10383 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 1864662 T10383 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 1864662 T10353 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:19802_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19802",
   [junit4]   2> 	  "collection":"collection1"}
   [junit4]   2> 1864663 T10353 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 1864663 T10383 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 1864663 T10353 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1864665 T10383 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 1864665 T10383 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1864666 T10353 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 1864667 T10382 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1864668 T10382 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 1865667 T10383 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1865667 T10383 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1865668 T10383 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1865668 T10383 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1865669 T10383 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1865670 T10383 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1865706 T10383 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1865711 T10383 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1865734 T10383 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1865775 T10383 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 1865892 T10383 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1865997 T10383 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1866021 T10383 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1866816 T10383 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1866846 T10383 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1866850 T10383 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1866864 T10383 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1866872 T10383 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1866881 T10383 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1866884 T10383 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1866885 T10383 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1866885 T10383 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1866886 T10383 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1866887 T10383 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1866887 T10383 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1866887 T10383 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1866888 T10383 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1866889 T10383 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@4eb85fa9
   [junit4]   2> 1866890 T10383 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1866890 T10383 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1866891 T10383 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1866891 T10383 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 7764FDC9C7D71437-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1866892 T10383 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=7, maxMergeAtOnceExplicit=4, maxMergedSegmentMB=23.8427734375, floorSegmentMB=0.431640625, forceMergeDeletesPctAllowed=10.29500517309118, segmentsPerTier=19.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.10316662867532474
   [junit4]   2> 1866893 T10383 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@62a8886d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2694deb5; maxCacheMB=0.8203125 maxMergeSizeMB=0.61328125)),segFN=segments_1,generation=1}
   [junit4]   2> 1866894 T10383 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1866904 T10383 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1866905 T10383 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1866905 T10383 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1866906 T10383 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1866906 T10383 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1866907 T10383 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1866907 T10383 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1866908 T10383 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1866908 T10383 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1866941 T10383 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1866945 T10383 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1866948 T10383 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1866952 T10383 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1866969 T10383 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/csv,/admin/segments,/schema,/admin/file,/admin/plugins,/config,/update/json/docs,/admin/luke,/admin/ping,/admin/logging,/get,/admin/properties,/replication,/admin/system,/update/json,/update,/admin/mbeans,/admin/threads,standard
   [junit4]   2> 1866973 T10383 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1866976 T10383 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1866977 T10383 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1866980 T10383 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1866980 T10383 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1866982 T10383 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@9420cc7
   [junit4]   2> 1866983 T10383 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@62a8886d lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2694deb5; maxCacheMB=0.8203125 maxMergeSizeMB=0.61328125)),segFN=segments_1,generation=1}
   [junit4]   2> 1866983 T10383 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1866984 T10383 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@406172b7[collection1] main
   [junit4]   2> 1866984 T10383 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1866987 T10383 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1866987 T10383 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1866989 T10383 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1866989 T10383 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1866989 T10383 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1866993 T10383 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1866993 T10383 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1866994 T10383 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1866994 T10383 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1866996 T10384 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@406172b7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1866997 T10383 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1866999 T10387 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:19802 collection:collection1 shard:shard1
   [junit4]   2> 1867000 T10326 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 1867001 T10326 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1867001 T10387 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 1867010 T10387 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1867012 T10352 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1867013 T10387 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1867013 T10387 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C934 name=collection1 org.apache.solr.core.SolrCore@666a6197 url=http://127.0.0.1:19802/collection1 node=127.0.0.1:19802_ C934_STATE=coll:collection1 core:collection1 props:{core=collection1, node_name=127.0.0.1:19802_, state=down, base_url=http://127.0.0.1:19802}
   [junit4]   2> 1867013 T10387 C:collection1 S:shard1 C934 P19802 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:19802/collection1/
   [junit4]   2> 1867014 T10353 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 1867014 T10387 C:collection1 S:shard1 C934 P19802 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1867015 T10387 C:collection1 S:shard1 C934 P19802 oasc.SyncStrategy.syncToMe http://127.0.0.1:19802/collection1/ has no replicas
   [junit4]   2> 1867015 T10387 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:19802/collection1/ shard1
   [junit4]   2> 1867016 T10387 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 1867016 T10353 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 1867017 T10382 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1867019 T10382 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 1867025 T10352 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1867029 T10353 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19802",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 1867030 T10353 oasco.ZkStateWri

[...truncated too long message...]

kins/lucene-data/enwiki.random.lines.txt -Dtests.locale=th -Dtests.timezone=SystemV/AST4 -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR    122s J1 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19834/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([7764FDC9C7D71437:FF30C213692B79CF]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:19834/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:19834 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 53 more
   [junit4]   2> 2745655 T7391 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 7764FDC9C7D71437-001
   [junit4]   2> 122157 T7390 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {text=PostingsFormat(name=MockRandom), a_t=PostingsFormat(name=LuceneFixedGap), intDefault=PostingsFormat(name=LuceneFixedGap), timestamp=PostingsFormat(name=LuceneFixedGap), multiDefault=PostingsFormat(name=LuceneVarGapFixedInterval), id=PostingsFormat(name=Direct), range_facet_l=PostingsFormat(name=LuceneFixedGap), _version_=PostingsFormat(name=LuceneFixedGap)}, docValues:{timestamp=DocValuesFormat(name=Direct)}, sim=DefaultSimilarity, locale=th, timezone=SystemV/AST4
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=313410072,total=529006592
   [junit4]   2> NOTE: All tests run in this JVM: [JSONWriterTest, PingRequestHandlerTest, OpenCloseCoreStressTest, DeleteShardTest, TestLFUCache, TermVectorComponentTest, TestCustomSort, TestSolr4Spatial, DistributedExpandComponentTest, DisMaxRequestHandlerTest, TestFuzzyAnalyzedSuggestions, TriLevelCompositeIdRoutingTest, AddBlockUpdateTest, ChaosMonkeySafeLeaderTest, AutoCommitTest, UpdateParamsTest, UniqFieldsUpdateProcessorFactoryTest, HdfsRecoveryZkTest, TestInitParams, DateRangeFieldTest, TestSolrDeletionPolicy1, TestConfig, TestZkChroot, TestStressRecovery, ConvertedLegacyTest, PathHierarchyTokenizerFactoryTest, EchoParamsTest, QueryElevationComponentTest, TestCloudInspectUtil, HdfsDirectoryTest, HdfsBasicDistributedZkTest, CircularListTest, FieldMutatingUpdateProcessorTest, FullSolrCloudDistribCmdsTest, SuggesterFSTTest, TestSweetSpotSimilarityFactory, HdfsDirectoryFactoryTest, TestOmitPositions, DistributedTermsComponentTest, TestCloudSchemaless, LoggingHandlerTest, TestDistribDocBasedVersion, VMParamsZkACLAndCredentialsProvidersTest, TestCollectionAPI, EnumFieldTest, SimplePostToolTest, SolrInfoMBeanTest, TestSolrDynamicMBean, CoreAdminCreateDiscoverTest, TestLMJelinekMercerSimilarityFactory, TestSolrConfigHandler, TestCoreDiscovery, TestDistributedMissingSort, SpatialRPTFieldTypeTest, TestFunctionQuery, ShardRoutingTest, LeaderElectionTest, ZkCLITest, TestRecovery, TestHashPartitioner, TestStressVersions, TestRangeQuery, SpellCheckComponentTest, BadIndexSchemaTest, TestSort, TestFiltering, BasicFunctionalityTest, TestBadConfig, OverseerCollectionProcessorTest, HighlighterTest, SoftAutoCommitTest, CurrencyFieldXmlFileTest, SpellCheckCollatorTest, TestFoldingMultitermQuery, SuggesterTSTTest, TestStressLucene, TestCSVLoader, WordBreakSolrSpellCheckerTest, SolrCoreCheckLockOnStartupTest, TestUpdate, TestWordDelimiterFilterFactory, DirectUpdateHandlerOptimizeTest, DefaultValueUpdateProcessorTest, StatelessScriptUpdateProcessorFactoryTest, SortByFunctionTest, DistanceFunctionTest, DebugComponentTest, TestQueryUtils, DirectSolrSpellCheckerTest, FileBasedSpellCheckerTest, MoreLikeThisHandlerTest, TestArbitraryIndexDir, TestJmxIntegration, QueryParsingTest, TestSolrIndexConfig, BadComponentTest, TestDocSet, MinimalSchemaTest, NotRequiredUniqueKeyTest, SpellPossibilityIteratorTest, TestCharFilters, TestPerFieldSimilarity, TestFastWriter, TestFastOutputStream, PluginInfoTest, PreAnalyzedFieldTest, TestLRUCache, TestUtils, ZkNodePropsTest, SystemInfoHandlerTest, UUIDFieldTest, TestRTGBase, TestSimpleTrackingShardHandler, CollectionsAPIAsyncDistributedZkTest, ExternalCollectionsTest, OverseerStatusTest, ReplicationFactorTest]
   [junit4] Completed on J1 in 122.43s, 1 test, 1 error <<< FAILURES!

[...truncated 325 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 479 suites, 1903 tests, 3 errors, 57 ignored (8 assumptions)

Total time: 192 minutes 2 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 922122980 bytes
Compression is 0.0%
Took 2 min 53 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 809 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/809/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:42792/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:42792/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([72E8D161E0485E39:FABCEEBB4EB433C1]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:42792 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:26811 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:26811 failed to respond
	at __randomizedtesting.SeedInfo.seed([72E8D161E0485E39:FABCEEBB4EB433C1]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:26811 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: 127.0.0.1:32568 failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:32568 failed to respond
	at __randomizedtesting.SeedInfo.seed([72E8D161E0485E39:FABCEEBB4EB433C1]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:32568 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 10143 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/init-core-data-001
   [junit4]   2> 484051 T3537 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 484060 T3537 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 484061 T3538 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 484161 T3537 oasc.ZkTestServer.run start zk server on port:42696
   [junit4]   2> 484162 T3537 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 484163 T3537 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 484168 T3545 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2ef9e11a name:ZooKeeperConnection Watcher:127.0.0.1:42696 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 484168 T3537 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 484169 T3537 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 484169 T3537 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 484174 T3537 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 484175 T3537 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 484177 T3548 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5f5e2085 name:ZooKeeperConnection Watcher:127.0.0.1:42696/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 484177 T3537 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 484178 T3537 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 484178 T3537 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 484180 T3537 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 484182 T3537 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 484183 T3537 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 484185 T3537 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 484185 T3537 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 484187 T3537 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 484188 T3537 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 484190 T3537 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 484190 T3537 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 484192 T3537 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 484193 T3537 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 484194 T3537 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 484195 T3537 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 484196 T3537 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 484197 T3537 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 484198 T3537 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 484199 T3537 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 484200 T3537 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 484201 T3537 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 484202 T3537 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 484203 T3537 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 484204 T3537 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 484205 T3537 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 484207 T3537 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 484207 T3537 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 485661 T3537 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/control-001/cores/collection1
   [junit4]   2> 485665 T3537 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 485669 T3537 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:42706
   [junit4]   2> 485669 T3537 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=42705, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/control-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/tempDir-001/control/data}
   [junit4]   2> 485670 T3537 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 485670 T3537 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/control-001/'
   [junit4]   2> 485712 T3537 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/control-001/solr.xml
   [junit4]   2> 485763 T3537 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/control-001/cores
   [junit4]   2> 485764 T3537 oasc.CoreContainer.<init> New CoreContainer 515143771
   [junit4]   2> 485764 T3537 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/control-001/]
   [junit4]   2> 485764 T3537 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/control-001/lib
   [junit4]   2> 485765 T3537 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/control-001/lib).
   [junit4]   2> 485776 T3537 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 485776 T3537 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 485776 T3537 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 485776 T3537 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 485777 T3537 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 485777 T3537 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 485777 T3537 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 485777 T3537 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 485777 T3537 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 485778 T3537 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 485778 T3537 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 485785 T3537 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 485786 T3537 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 485786 T3537 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 485786 T3537 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 485787 T3537 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:42696/solr
   [junit4]   2> 485787 T3537 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 485787 T3537 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 485788 T3537 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 485789 T3560 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@79ba4819 name:ZooKeeperConnection Watcher:127.0.0.1:42696 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 485790 T3537 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 485791 T3537 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 485792 T3537 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 485793 T3563 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@24e2619b name:ZooKeeperConnection Watcher:127.0.0.1:42696/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 485793 T3537 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 485794 T3537 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 485796 T3537 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 485797 T3537 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 485799 T3537 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 485800 T3537 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 485802 T3537 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 485803 T3537 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 485804 T3537 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 485805 T3537 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42705_
   [junit4]   2> 485805 T3537 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:42705_
   [junit4]   2> 485806 T3537 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 485807 T3537 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 485808 T3537 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 485809 T3537 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:42705_
   [junit4]   2> 485810 T3537 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 485811 T3537 oasc.Overseer.start Overseer (id=93607325298589699-127.0.0.1:42705_-n_0000000000) starting
   [junit4]   2> 485812 T3537 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 485816 T3537 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 485817 T3565 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 485817 T3537 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 485817 T3564 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 485819 T3537 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/control-001/cores
   [junit4]   2> 485822 T3537 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/control-001/cores/collection1/, shard=, dataDir=data/, collection=control_collection, transient=false, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/control-001/cores/collection1, coreNodeName=, loadOnStartup=true, config=solrconfig.xml}
   [junit4]   2> 485822 T3537 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/control-001/cores/collection1/
   [junit4]   2> 485822 T3537 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 485823 T3567 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 485823 T3567 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 485824 T3567 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 485824 T3563 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 485824 T3564 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:42705_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42705"} current state version: 0
   [junit4]   2> 485825 T3564 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:42705_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42705"}
   [junit4]   2> 485825 T3564 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 485825 T3564 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 485826 T3563 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 486824 T3567 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 486825 T3567 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 486826 T3567 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 486826 T3567 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 486826 T3567 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 486827 T3567 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/control-001/cores/collection1/'
   [junit4]   2> 486845 T3567 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 486848 T3567 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 486860 T3567 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 486884 T3567 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 486939 T3567 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 486940 T3567 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 486954 T3567 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 487414 T3567 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 487431 T3567 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 487435 T3567 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 487448 T3567 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 487453 T3567 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 487457 T3567 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 487458 T3567 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 487458 T3567 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 487459 T3567 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 487459 T3567 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 487460 T3567 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 487460 T3567 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 487460 T3567 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 487461 T3567 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/control-001/cores/collection1/data/
   [junit4]   2> 487461 T3567 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@59611d68
   [junit4]   2> 487462 T3567 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/control-001/cores/collection1/data
   [junit4]   2> 487462 T3567 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/control-001/cores/collection1/data/index/
   [junit4]   2> 487462 T3567 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 487464 T3567 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/control-001/cores/collection1/data/index
   [junit4]   2> 487465 T3567 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=27, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.13366184610853554]
   [junit4]   2> 487470 T3567 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(SimpleFSDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/index-SimpleFSDirectory-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@327b892a),segFN=segments_1,generation=1}
   [junit4]   2> 487470 T3567 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 487478 T3567 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 487479 T3567 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 487479 T3567 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 487479 T3567 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 487479 T3567 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 487480 T3567 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 487480 T3567 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 487481 T3567 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 487481 T3567 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 487512 T3567 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 487515 T3567 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 487518 T3567 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 487520 T3567 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 487528 T3567 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/plugins,/get,/update/json/docs,/schema,/admin/segments,/config,/admin/system,/admin/threads,/admin/file,/admin/properties,/admin/mbeans,/admin/ping,/update,/admin/logging,/update/csv,/replication,/update/json,standard,/admin/luke
   [junit4]   2> 487530 T3567 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 487532 T3567 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 487532 T3567 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 487534 T3567 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 487534 T3567 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 487535 T3567 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=37, maxMergeAtOnceExplicit=40, maxMergedSegmentMB=32.2431640625, floorSegmentMB=1.43359375, forceMergeDeletesPctAllowed=20.857387966766833, segmentsPerTier=40.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4527166887758399
   [junit4]   2> 487538 T3567 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(SimpleFSDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/index-SimpleFSDirectory-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@327b892a),segFN=segments_1,generation=1}
   [junit4]   2> 487538 T3567 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 487539 T3567 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4253288e[collection1] main
   [junit4]   2> 487539 T3567 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 487540 T3567 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 487540 T3567 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 487541 T3567 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 487541 T3567 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 487541 T3567 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 487542 T3567 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 487542 T3567 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 487542 T3567 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 487543 T3567 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 487544 T3568 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4253288e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 487545 T3567 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 487546 T3571 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:42705 collection:control_collection shard:shard1
   [junit4]   2> 487546 T3537 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 487546 T3537 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 487547 T3571 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 487552 T3571 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 487552 T3537 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 487553 T3537 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 487554 T3563 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 487554 T3571 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 487554 T3571 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C505 name=collection1 org.apache.solr.core.SolrCore@1c028aa url=http://127.0.0.1:42705/collection1 node=127.0.0.1:42705_ C505_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, node_name=127.0.0.1:42705_, base_url=http://127.0.0.1:42705}
   [junit4]   2> 487555 T3571 C:control_collection S:shard1 C505 P42705 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:42705/collection1/
   [junit4]   2> 487555 T3571 C:control_collection S:shard1 C505 P42705 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 487555 T3575 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6015ea9b name:ZooKeeperConnection Watcher:127.0.0.1:42696/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 487555 T3564 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 487555 T3571 C:control_collection S:shard1 C505 P42705 oasc.SyncStrategy.syncToMe http://127.0.0.1:42705/collection1/ has no replicas
   [junit4]   2> 487556 T3537 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 487556 T3571 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:42705/collection1/ shard1
   [junit4]   2> 487557 T3571 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 487557 T3537 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 487557 T3563 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 487557 T3537 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 487561 T3563 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 487562 T3537 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 487562 T3564 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42705",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 487665 T3563 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 487665 T3575 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 487714 T3571 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:42705/collection1/ and leader is http://127.0.0.1:42705/collection1/
   [junit4]   2> 487715 T3571 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:42705
   [junit4]   2> 487715 T3571 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 487715 T3571 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 487716 T3571 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 487717 T3563 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 487718 T3564 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:42705_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42705"} current state version: 3
   [junit4]   2> 487719 T3564 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:42705_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42705"}
   [junit4]   2> 487822 T3575 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 487822 T3563 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 489617 T3537 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/shard-1-001/cores/collection1
   [junit4]   2> 489620 T3537 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/shard-1-001
   [junit4]   2> 489621 T3537 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 489624 T3537 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:42719
   [junit4]   2> 489625 T3537 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=42718, solrconfig=solrconfig.xml, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/tempDir-001/jetty1}
   [junit4]   2> 489626 T3537 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 489626 T3537 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/shard-1-001/'
   [junit4]   2> 489679 T3537 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/shard-1-001/solr.xml
   [junit4]   2> 489741 T3537 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/shard-1-001/cores
   [junit4]   2> 489741 T3537 oasc.CoreContainer.<init> New CoreContainer 288930283
   [junit4]   2> 489742 T3537 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/shard-1-001/]
   [junit4]   2> 489742 T3537 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/shard-1-001/lib
   [junit4]   2> 489743 T3537 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/shard-1-001/lib).
   [junit4]   2> 489757 T3537 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 489757 T3537 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 489758 T3537 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 489758 T3537 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 489759 T3537 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 489759 T3537 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 489759 T3537 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 489760 T3537 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 489760 T3537 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 489760 T3537 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 489761 T3537 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 489769 T3537 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 489770 T3537 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 489770 T3537 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 489771 T3537 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 489771 T3537 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:42696/solr
   [junit4]   2> 489771 T3537 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 489772 T3537 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 489773 T3537 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 489775 T3587 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7cb4fb1 name:ZooKeeperConnection Watcher:127.0.0.1:42696 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 489775 T3537 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 489776 T3537 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 489778 T3537 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 489780 T3590 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3199c0ae name:ZooKeeperConnection Watcher:127.0.0.1:42696/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 489780 T3537 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 489784 T3537 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 490789 T3537 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:42718_
   [junit4]   2> 490790 T3537 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:42718_
   [junit4]   2> 490792 T3537 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 490794 T3537 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/shard-1-001/cores
   [junit4]   2> 490797 T3537 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, schema=schema.xml, config=solrconfig.xml, shard=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/shard-1-001/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/shard-1-001/cores/collection1/, transient=false, loadOnStartup=true, name=collection1, collection=collection1, coreNodeName=}
   [junit4]   2> 490798 T3537 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/shard-1-001/cores/collection1/
   [junit4]   2> 490798 T3537 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 490799 T3591 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 490799 T3591 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 490800 T3563 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 490800 T3591 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 490801 T3564 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:42718_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42718"} current state version: 4
   [junit4]   2> 490801 T3564 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:42718_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42718"}
   [junit4]   2> 490802 T3564 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 490802 T3564 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 490803 T3563 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 490803 T3575 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 490803 T3590 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 491801 T3591 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 491802 T3591 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 491803 T3591 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 491803 T3591 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 491804 T3591 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 491804 T3591 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/shard-1-001/cores/collection1/'
   [junit4]   2> 491823 T3591 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 491826 T3591 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 491838 T3591 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 491861 T3591 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 491912 T3591 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 491914 T3591 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 491923 T3591 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 492242 T3591 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 492259 T3591 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 492263 T3591 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 492277 T3591 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 492281 T3591 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 492286 T3591 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 492286 T3591 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 492287 T3591 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 492287 T3591 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 492288 T3591 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 492288 T3591 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 492288 T3591 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 492288 T3591 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 492289 T3591 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 492289 T3591 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@59611d68
   [junit4]   2> 492290 T3591 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/shard-1-001/cores/collection1/data
   [junit4]   2> 492290 T3591 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 492291 T3591 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 492293 T3591 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 492294 T3591 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=27, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.13366184610853554]
   [junit4]   2> 492300 T3591 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(SimpleFSDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/index-SimpleFSDirectory-002 lockFactory=org.apache.lucene.store.NativeFSLockFactory@327b892a),segFN=segments_1,generation=1}
   [junit4]   2> 492300 T3591 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 492310 T3591 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 492310 T3591 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 492310 T3591 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 492311 T3591 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 492311 T3591 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 492311 T3591 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 492312 T3591 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 492312 T3591 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 492312 T3591 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 492341 T3591 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 492344 T3591 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 492347 T3591 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 492349 T3591 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 492360 T3591 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/plugins,/get,/update/json/docs,/schema,/admin/segments,/config,/admin/system,/admin/threads,/admin/file,/admin/properties,/admin/mbeans,/admin/ping,/update,/admin/logging,/update/csv,/replication,/update/json,standard,/admin/luke
   [junit4]   2> 492362 T3591 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 492365 T3591 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 492365 T3591 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 492367 T3591 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 492367 T3591 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 492368 T3591 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=37, maxMergeAtOnceExplicit=40, maxMergedSegmentMB=32.2431640625, floorSegmentMB=1.43359375, forceMergeDeletesPctAllowed=20.857387966766833, segmentsPerTier=40.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4527166887758399
   [junit4]   2> 492372 T3591 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(SimpleFSDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 72E8D161E0485E39-001/index-SimpleFSDirectory-002 lockFactory=org.apache.lucene.store.NativeFSLockFactory@327b892a),segFN=segments_1,generation=1}
   [junit4]   2> 492372 T3591 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 492372 T3591 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@5206c1d0[collection1] main
   [junit4]   2> 492373 T3591 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 492374 T3591 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 492374 T3591 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 492374 T3591 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 492374 T3591 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 492375 T3591 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 492375 T3591 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 492376 T3591 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 492376 T3591 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 492376 T3591 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 492377 T3592 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5206c1d0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 492378 T3591 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 492379 T3595 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:42718 collection:collection1 shard:shard1
   [junit4]   2> 492379 T3537 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 492379 T3537 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 492379 T3595 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 492383 T3595 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 492384 T3563 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 492384 T3595 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 492385 T3595 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C506 name=collection1 org.apache.solr.core.SolrCore@7db5208b url=http://127.0.0.1:42718/collection1 node=127.0.0.1:42718_ C506_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, node_name=127.0.0.1:42718_, base_url=http://127.0.0.1:42718}
   [junit4]   2> 492385 T3595 C:collection1 S:shard1 C506 P42718 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:42718/collection1/
   [junit4]   2> 492385 T3564 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 492385 T3595 C:collection1 S:shard1 C506 P42718 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 492385 T3595 C:collection1 S:shard1 C506 P42718 oasc.SyncStrategy.syncToMe http://127.0.0.1:42718/collection1/ has no replicas
   [junit4]   2> 492386 T3595 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:42718/collection1/ shard1
   [junit4]   2> 492386 T3595 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 492386 T3563 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 492386 T3575 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 492386 T3590 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 492389 T3563 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 492390 T3564 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42718",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 492493 T3590 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 492493 T3575 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 492493 T3563 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 492542 T3595 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:42718/collection1/ and leader is http://127.0.0.1:42718/collection1/
   [junit4]   2> 492543 T3595 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:42718
   [junit4]   2> 492543 T3595 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 492543 T3595 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 492543 T3595 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 492545 T3563 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 492545 T3564 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:42718_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:42718"} current state version: 7
   [junit4]   2> 492546 T3564 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]  

[...truncated too long message...]

rowable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: 127.0.0.1:32568 failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([72E8D161E0485E39:FABCEEBB4EB433C1]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:32568 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 2643054 T13423 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest 72E8D161E0485E39-001
   [junit4]   2> 73617 T13422 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=is_IS, timezone=Mexico/BajaNorte
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=177730296,total=402128896
   [junit4]   2> NOTE: All tests run in this JVM: [LeaderElectionTest, TestInitQParser, HdfsChaosMonkeySafeLeaderTest, TestArbitraryIndexDir, TestIntervalFaceting, TestSystemIdResolver, ClusterStateUpdateTest, TestLRUStatsCache, IndexSchemaRuntimeFieldTest, CSVRequestHandlerTest, BadIndexSchemaTest, RollingRestartTest, TestUtils, InfoHandlerTest, TestSolrQueryParserResource, SoftAutoCommitTest, SpellPossibilityIteratorTest, TestFieldCollectionResource, ZkCLITest, TestNonDefinedSimilarityFactory, DeleteLastCustomShardedReplicaTest, TestClusterStateMutator, TestMaxScoreQueryParser, TestPseudoReturnFields, TestJmxMonitoredMap, TestCSVLoader, TestCodecSupport, NumericFieldsTest, TestObjectReleaseTracker, TermsComponentTest, BinaryUpdateRequestHandlerTest, TestConfigReload, TestShardHandlerFactory, TestDistributedGrouping, DistributedIntervalFacetingTest, DistributedMLTComponentTest, CoreAdminRequestStatusTest, TestLMDirichletSimilarityFactory, SpellingQueryConverterTest, JsonLoaderTest, XsltUpdateRequestHandlerTest, ResponseLogComponentTest, TestWordDelimiterFilterFactory, DateMathParserTest, TestManagedSynonymFilterFactory, TestRebalanceLeaders, TermVectorComponentDistributedTest, DocValuesMultiTest, ScriptEngineTest, SolrCoreTest, CurrencyFieldXmlFileTest, TestSchemaManager, ClusterStateTest, TestTrackingShardHandlerFactory, TestIndexingPerformance, UnloadDistributedZkTest, UUIDUpdateProcessorFallbackTest, QueryResultKeyTest, TestCursorMarkWithoutUniqueKey, LeaderElectionIntegrationTest, ShardRoutingTest, FullSolrCloudDistribCmdsTest, TestRandomFaceting, ZkSolrClientTest, TestRandomDVFaceting, ShardRoutingCustomTest, TestFaceting, TestRecovery, ZkControllerTest, TestRealTimeGet, DistributedTermsComponentTest, TestSolr4Spatial, SolrCmdDistributorTest, QueryElevationComponentTest, TestSort, TestFiltering, TestLazyCores, TestIndexSearcher, SimplePostToolTest, TestCoreDiscovery, SignatureUpdateProcessorFactoryTest, TestExtendedDismaxParser, TestFoldingMultitermQuery, DocValuesTest, SpatialFilterTest, PolyFieldTest, SchemaVersionSpecificBehaviorTest, StatelessScriptUpdateProcessorFactoryTest, TestSolrDeletionPolicy1, SolrInfoMBeanTest, IndexBasedSpellCheckerTest, LukeRequestHandlerTest, TestWriterPerf, TestReversedWildcardFilterFactory, DocumentAnalysisRequestHandlerTest, FileBasedSpellCheckerTest, TestValueSourceCache, TermVectorComponentTest, RequiredFieldsTest, LoggingHandlerTest, TestJmxIntegration, UpdateRequestProcessorFactoryTest, QueryParsingTest, TestComponentsName, SearchHandlerTest, SOLR749Test, AlternateDirectoryTest, CopyFieldTest, TestMergePolicyConfig, TestSolrDeletionPolicy2, MultiTermTest, SampleTest, TestDocSet, TestConfig, OutputWriterTest, ExternalFileFieldSortTest, TestSolrCoreProperties, TestPostingsSolrHighlighter, DirectSolrConnectionTest, TestSweetSpotSimilarityFactory, TestDFRSimilarityFactory, TestPerFieldSimilarity, TestIBSimilarityFactory, TestDefaultSimilarityFactory, ResourceLoaderTest, OpenExchangeRatesOrgProviderTest, URLClassifyProcessorTest, TestFastLRUCache, PreAnalyzedFieldTest, TestSuggestSpellingConverter, DateFieldTest, DOMUtilTest, TestSolrJ, TestLRUCache, SystemInfoHandlerTest, UUIDFieldTest, FileUtilsTest, CircularListTest, TestRTGBase, SolrTestCaseJ4Test, TestDistributedMissingSort, TestHighlightDedupGrouping, AssignTest, AsyncMigrateRouteKeyTest, DistribDocExpirationUpdateProcessorTest, DistributedQueueTest, ExternalCollectionsTest, MultiThreadedOCPTest, OverseerStatusTest, RemoteQueryErrorTest, ReplicationFactorTest]
   [junit4] Completed on J3 in 74.14s, 1 test, 1 error <<< FAILURES!

[...truncated 306 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 57 ignored (8 assumptions)

Total time: 161 minutes 30 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 893986177 bytes
Compression is 0.0%
Took 2 min 48 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 808 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/808/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:25050/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:25050/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([9BEF27E0B7716FA:81EACDA4A58B7B02]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:25050 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31718/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31718/collection1
	at __randomizedtesting.SeedInfo.seed([9BEF27E0B7716FA:81EACDA4A58B7B02]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31718/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:31718 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:25401/mv_/l/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:25401/mv_/l/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([9BEF27E0B7716FA:81EACDA4A58B7B02]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:25401/mv_/l/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:25401 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 10066 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/init-core-data-001
   [junit4]   2> 151422 T436 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 151426 T436 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 151427 T437 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 151528 T436 oasc.ZkTestServer.run start zk server on port:31693
   [junit4]   2> 151529 T436 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 151529 T436 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 151533 T444 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@31f50e1f name:ZooKeeperConnection Watcher:127.0.0.1:31693 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 151534 T436 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 151534 T436 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 151534 T436 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 151537 T436 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 151538 T436 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 151539 T447 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@22d59577 name:ZooKeeperConnection Watcher:127.0.0.1:31693/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 151540 T436 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 151540 T436 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 151540 T436 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 151543 T436 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 151545 T436 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 151547 T436 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 151549 T436 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 151549 T436 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 151552 T436 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 151553 T436 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 151555 T436 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 151555 T436 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 151558 T436 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 151558 T436 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 151560 T436 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 151560 T436 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 151563 T436 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 151563 T436 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 151565 T436 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 151565 T436 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 151567 T436 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 151568 T436 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 151570 T436 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 151570 T436 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 151572 T436 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 151572 T436 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 151574 T436 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 151575 T436 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 153043 T436 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/control-001/cores/collection1
   [junit4]   2> 153047 T436 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 153051 T436 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:31699
   [junit4]   2> 153051 T436 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/control-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/tempDir-001/control/data, hostPort=31698}
   [junit4]   2> 153052 T436 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@5fb271f1
   [junit4]   2> 153052 T436 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/control-001/'
   [junit4]   2> 153101 T436 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/control-001/solr.xml
   [junit4]   2> 153163 T436 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/control-001/cores
   [junit4]   2> 153163 T436 oasc.CoreContainer.<init> New CoreContainer 1196284276
   [junit4]   2> 153164 T436 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/control-001/]
   [junit4]   2> 153164 T436 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/control-001/lib
   [junit4]   2> 153165 T436 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/control-001/lib).
   [junit4]   2> 153179 T436 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 153179 T436 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 153179 T436 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 153179 T436 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 153180 T436 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 153180 T436 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 153180 T436 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 153181 T436 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 153181 T436 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 153181 T436 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 153181 T436 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 153193 T436 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 153194 T436 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 153194 T436 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 153195 T436 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 153195 T436 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:31693/solr
   [junit4]   2> 153195 T436 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 153196 T436 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 153197 T436 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 153199 T459 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@35bc2b0a name:ZooKeeperConnection Watcher:127.0.0.1:31693 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 153199 T436 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 153200 T436 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 153203 T436 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 153204 T462 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@36ae4b6f name:ZooKeeperConnection Watcher:127.0.0.1:31693/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 153205 T436 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 153207 T436 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 153210 T436 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 153213 T436 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 153215 T436 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 153218 T436 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 153221 T436 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 153223 T436 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 153225 T436 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 153226 T436 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:31698_
   [junit4]   2> 153227 T436 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:31698_
   [junit4]   2> 153229 T436 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 153231 T436 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 153233 T436 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 153235 T436 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:31698_
   [junit4]   2> 153235 T436 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 153237 T436 oasc.Overseer.start Overseer (id=93603043649323011-127.0.0.1:31698_-n_0000000000) starting
   [junit4]   2> 153240 T436 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 153248 T436 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 153249 T464 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 153249 T436 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 153249 T463 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 153254 T436 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/control-001/cores
   [junit4]   2> 153257 T436 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, shard=, name=collection1, schema=schema.xml, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/control-001/cores/collection1/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/control-001/cores/collection1, loadOnStartup=true, config=solrconfig.xml, coreNodeName=, collection=control_collection}
   [junit4]   2> 153258 T436 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/control-001/cores/collection1/
   [junit4]   2> 153258 T436 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 153259 T466 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 153259 T466 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 153260 T462 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 153260 T466 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 153261 T463 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31698",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:31698_"} current state version: 0
   [junit4]   2> 153262 T463 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31698",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:31698_"}
   [junit4]   2> 153262 T463 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 153263 T463 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 153264 T462 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 154262 T466 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 154263 T466 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 154263 T466 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 154264 T466 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 154264 T466 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 154265 T466 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/control-001/cores/collection1/'
   [junit4]   2> 154278 T466 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 154280 T466 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 154291 T466 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 154309 T466 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 154351 T466 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 154352 T466 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 154360 T466 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 154716 T466 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 154733 T466 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 154735 T466 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 154754 T466 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 154758 T466 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 154762 T466 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 154763 T466 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 154764 T466 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 154764 T466 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 154765 T466 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 154765 T466 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 154765 T466 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 154766 T466 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 154766 T466 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/control-001/cores/collection1/data/
   [junit4]   2> 154766 T466 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@18ddf9a
   [junit4]   2> 154767 T466 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/control-001/cores/collection1/data
   [junit4]   2> 154767 T466 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/control-001/cores/collection1/data/index/
   [junit4]   2> 154767 T466 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 154768 T466 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/control-001/cores/collection1/data/index
   [junit4]   2> 154768 T466 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=37, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7223085488896641]
   [junit4]   2> 154769 T466 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5e4d84ec lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3351d252),segFN=segments_1,generation=1}
   [junit4]   2> 154769 T466 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 154775 T466 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 154887 T466 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 154887 T466 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 154888 T466 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 154888 T466 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 154888 T466 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 154889 T466 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 154889 T466 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 154889 T466 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 154918 T466 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 154921 T466 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 154924 T466 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 154927 T466 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 154933 T466 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/luke,/admin/system,/admin/plugins,/admin/file,/admin/properties,/admin/ping,/update/json,/admin/segments,/update,/update/json/docs,/schema,standard,/replication,/admin/logging,/admin/mbeans,/get,/admin/threads,/config,/update/csv
   [junit4]   2> 154936 T466 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 154938 T466 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 154938 T466 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 154940 T466 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 154940 T466 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 154941 T466 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=20, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=98.2001953125, floorSegmentMB=1.9306640625, forceMergeDeletesPctAllowed=14.834311499927924, segmentsPerTier=27.0, maxCFSSegmentSizeMB=1.3037109375, noCFSRatio=0.3028406928879407
   [junit4]   2> 154942 T466 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5e4d84ec lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3351d252),segFN=segments_1,generation=1}
   [junit4]   2> 154942 T466 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 154943 T466 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@17739f5[collection1] main
   [junit4]   2> 154943 T466 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 154944 T466 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 154944 T466 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 154945 T466 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 154945 T466 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 154945 T466 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 154946 T466 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 154946 T466 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 154947 T466 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 154947 T466 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 154948 T467 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@17739f5[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 154948 T466 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 154949 T470 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:31698 collection:control_collection shard:shard1
   [junit4]   2> 154950 T436 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 154950 T436 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 154951 T470 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 154955 T436 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 154956 T436 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 154958 T470 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 154958 T474 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@f57e027 name:ZooKeeperConnection Watcher:127.0.0.1:31693/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 154959 T436 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 154959 T436 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 154960 T436 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 154960 T462 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 154961 T470 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 154961 T470 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C100 name=collection1 org.apache.solr.core.SolrCore@15bb1524 url=http://127.0.0.1:31698/collection1 node=127.0.0.1:31698_ C100_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:31698, core=collection1, state=down, node_name=127.0.0.1:31698_}
   [junit4]   2> 154961 T470 C:control_collection S:shard1 C100 P31698 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:31698/collection1/
   [junit4]   2> 154962 T463 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 154962 T470 C:control_collection S:shard1 C100 P31698 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 154963 T470 C:control_collection S:shard1 C100 P31698 oasc.SyncStrategy.syncToMe http://127.0.0.1:31698/collection1/ has no replicas
   [junit4]   2> 154963 T470 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:31698/collection1/ shard1
   [junit4]   2> 154963 T470 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 154964 T462 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 154964 T474 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 154965 T436 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 154970 T462 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 154971 T463 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31698",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 155076 T462 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 155076 T474 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 155123 T470 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:31698/collection1/ and leader is http://127.0.0.1:31698/collection1/
   [junit4]   2> 155124 T470 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:31698
   [junit4]   2> 155124 T470 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 155124 T470 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 155124 T470 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 155126 T462 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 155128 T463 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31698",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:31698_"} current state version: 3
   [junit4]   2> 155129 T463 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31698",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:31698_"}
   [junit4]   2> 155234 T462 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 155234 T474 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 156504 T436 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/shard-1-001/cores/collection1
   [junit4]   2> 156506 T436 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/shard-1-001
   [junit4]   2> 156507 T436 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 156509 T436 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:31719
   [junit4]   2> 156510 T436 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/tempDir-001/jetty1, hostContext=/, solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/shard-1-001/cores, hostPort=31718}
   [junit4]   2> 156510 T436 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@5fb271f1
   [junit4]   2> 156510 T436 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/shard-1-001/'
   [junit4]   2> 156543 T436 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/shard-1-001/solr.xml
   [junit4]   2> 156583 T436 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/shard-1-001/cores
   [junit4]   2> 156584 T436 oasc.CoreContainer.<init> New CoreContainer 1767399124
   [junit4]   2> 156584 T436 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/shard-1-001/]
   [junit4]   2> 156584 T436 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/shard-1-001/lib
   [junit4]   2> 156585 T436 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/shard-1-001/lib).
   [junit4]   2> 156594 T436 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 156594 T436 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 156595 T436 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 156595 T436 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 156595 T436 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 156595 T436 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 156595 T436 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 156596 T436 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 156596 T436 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 156596 T436 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 156596 T436 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 156604 T436 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 156604 T436 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 156605 T436 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 156605 T436 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 156605 T436 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:31693/solr
   [junit4]   2> 156605 T436 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 156606 T436 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 156606 T436 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 156608 T486 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7cf5ae82 name:ZooKeeperConnection Watcher:127.0.0.1:31693 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 156609 T436 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 156609 T436 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 156611 T436 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 156613 T489 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@73d697b3 name:ZooKeeperConnection Watcher:127.0.0.1:31693/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 156614 T436 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 156620 T436 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 157626 T436 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:31718_
   [junit4]   2> 157627 T436 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:31718_
   [junit4]   2> 157632 T436 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 157634 T436 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/shard-1-001/cores
   [junit4]   2> 157636 T436 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/shard-1-001/cores/collection1/, name=collection1, coreNodeName=, dataDir=data/, loadOnStartup=true, transient=false, schema=schema.xml, shard=, collection=collection1, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/shard-1-001/cores/collection1}
   [junit4]   2> 157636 T436 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/shard-1-001/cores/collection1/
   [junit4]   2> 157637 T436 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 157638 T490 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 157638 T490 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 157639 T462 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 157639 T490 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 157640 T463 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31718",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:31718_"} current state version: 4
   [junit4]   2> 157641 T463 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31718",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:31718_"}
   [junit4]   2> 157641 T463 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 157641 T463 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 157643 T462 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 157643 T489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 157643 T474 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 158640 T490 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 158641 T490 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 158642 T490 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 158642 T490 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 158643 T490 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 158644 T490 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/shard-1-001/cores/collection1/'
   [junit4]   2> 158662 T490 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 158667 T490 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 158684 T490 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 158712 T490 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 158774 T490 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 158776 T490 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 158785 T490 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 159153 T490 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 159172 T490 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 159175 T490 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 159188 T490 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 159193 T490 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 159197 T490 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 159198 T490 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 159198 T490 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 159198 T490 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 159199 T490 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 159199 T490 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 159200 T490 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 159200 T490 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 159200 T490 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 159201 T490 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@18ddf9a
   [junit4]   2> 159201 T490 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/shard-1-001/cores/collection1/data
   [junit4]   2> 159202 T490 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 159202 T490 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 159202 T490 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9BEF27E0B7716FA-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 159202 T490 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=37, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7223085488896641]
   [junit4]   2> 159203 T490 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3e8dfdb7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5c0c1976),segFN=segments_1,generation=1}
   [junit4]   2> 159203 T490 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 159210 T490 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 159211 T490 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 159211 T490 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 159211 T490 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 159211 T490 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 159211 T490 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 159212 T490 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 159212 T490 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 159212 T490 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 159237 T490 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 159239 T490 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 159242 T490 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 159244 T490 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 159252 T490 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/luke,/admin/system,/admin/plugins,/admin/file,/admin/properties,/admin/ping,/update/json,/admin/segments,/update,/update/json/docs,/schema,standard,/replication,/admin/logging,/admin/mbeans,/get,/admin/threads,/config,/update/csv
   [junit4]   2> 159254 T490 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 159255 T490 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 159255 T490 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 159257 T490 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 159257 T490 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 159258 T490 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=20, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=98.2001953125, floorSegmentMB=1.9306640625, forceMergeDeletesPctAllowed=14.834311499927924, segmentsPerTier=27.0, maxCFSSegmentSizeMB=1.3037109375, noCFSRatio=0.3028406928879407
   [junit4]   2> 159258 T490 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3e8dfdb7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5c0c1976),segFN=segments_1,generation=1}
   [junit4]   2> 159258 T490 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 159259 T490 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4ad98f01[collection1] main
   [junit4]   2> 159259 T490 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 159260 T490 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 159260 T490 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 159260 T490 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 159260 T490 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 159261 T490 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 159261 T490 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 159261 T490 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 159261 T490 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 159262 T490 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 159263 T491 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4ad98f01[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 159263 T490 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 159264 T494 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:31718 collection:collection1 shard:shard1
   [junit4]   2> 159264 T436 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 159264 T436 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 159265 T494 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 159272 T494 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 159274 T462 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 159275 T494 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 159275 T494 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C101 name=collection1 org.apache.solr.core.SolrCore@45ee63c2 url=http://127.0.0.1:31718/collection1 node=127.0.0.1:31718_ C101_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:31718, core=collection1, state=down, node_name=127.0.0.1:31718_}
   [junit4]   2> 159275 T494 C:collection1 S:shard1 C101 P31718 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:31718/collection1/
   [junit4]   2> 159276 T494 C:collection1 S:shard1 C101 P31718 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 159276 T463 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 159276 T494 C:collection1 S:shard1 C101 P31718 oasc.SyncStrategy.syncToMe http://127.0.0.1:31718/collection1/ has no replicas
   [junit4]   2> 159277 T494 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:31718/collection1/ shard1
   [junit4]   2> 159277 T494 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 159279 T474 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 159279 T462 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 159279 T489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 159285 T462 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 159287 T463 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31718",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 159393 T474 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 159393 T489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 159393 T462 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 159439 T494 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:31718/collection1/ and leader is http://127.0.0.1:31718/collection1/
   [junit4]   2> 159440 T494 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:31718
   [junit4]   2> 159440 T494 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 159441 T494 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 159441 T494 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 159443 T462 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 159445 T463 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31718",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:31718_"} current state version: 7
   [junit4]   2> 159446 T463 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:31718",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:31718_"}
   [junit4]   2> 159553 T474 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 159553 T489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 159554 T462 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 160965 T436 oas.SolrTest

[...truncated too long message...]

locale=zh_TW -Dtests.timezone=Asia/Novokuznetsk -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   78.7s J1 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:25401/mv_/l/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([9BEF27E0B7716FA:81EACDA4A58B7B02]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:25401/mv_/l/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:25401 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 53 more
   [junit4]   2> 2747486 T13275 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 9BEF27E0B7716FA-001
   [junit4]   2> 78730 T13274 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {intDefault=PostingsFormat(name=MockRandom), id=PostingsFormat(name=Direct), _version_=PostingsFormat(name=MockRandom), a_t=PostingsFormat(name=Memory doPackFST= true), multiDefault=PostingsFormat(name=Memory doPackFST= true), timestamp=PostingsFormat(name=Memory doPackFST= true), range_facet_l=PostingsFormat(name=Memory doPackFST= true), text=FSTOrd50}, docValues:{timestamp=DocValuesFormat(name=Asserting)}, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=zh_TW, timezone=Asia/Novokuznetsk
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=228392208,total=445644800
   [junit4]   2> NOTE: All tests run in this JVM: [ResourceLoaderTest, CoreAdminCreateDiscoverTest, TestSolr4Spatial, TestSolrDeletionPolicy1, TestFuzzyAnalyzedSuggestions, TestOmitPositions, BasicZkTest, SegmentsInfoRequestHandlerTest, TestSearchPerf, DistributedExpandComponentTest, TriLevelCompositeIdRoutingTest, LoggingHandlerTest, TestFieldResource, TestDynamicLoading, PreAnalyzedUpdateProcessorTest, TestJsonFacets, FieldAnalysisRequestHandlerTest, TestStressLucene, HdfsSyncSliceTest, TermVectorComponentTest, TestMiniSolrCloudCluster, TestDocumentBuilder, EchoParamsTest, DirectSolrConnectionTest, SuggesterFSTTest, TestPhraseSuggestions, PluginInfoTest, TestLMJelinekMercerSimilarityFactory, SolrInfoMBeanTest, TestExactSharedStatsCache, TestFunctionQuery, SuggesterTSTTest, TestCollectionAPI, DistribJoinFromCollectionTest, StressHdfsTest, TestLFUCache, DocumentAnalysisRequestHandlerTest, TestBinaryResponseWriter, UniqFieldsUpdateProcessorFactoryTest, TestDistributedMissingSort, TestHashQParserPlugin, SortByFunctionTest, TestEmbeddedSolrServerConstructors, TestCloudSchemaless, EnumFieldTest, TestReRankQParserPlugin, AddBlockUpdateTest, TestSolrConfigHandlerCloud, CloudExitableDirectoryReaderTest, TestZkChroot, TestNoOpRegenerator, TestSweetSpotSimilarityFactory, HdfsCollectionsAPIDistributedZkTest, DisMaxRequestHandlerTest, DeleteShardTest, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, BasicDistributedZk2Test, LeaderElectionIntegrationTest, ShardRoutingTest, RecoveryZkTest, FullSolrCloudDistribCmdsTest, TestRandomFaceting, LeaderElectionTest, TestDistributedGrouping, TermVectorComponentDistributedTest, TestStressReorder, TestStressVersions, TestCoreContainer, SimpleFacetsTest, SolrCoreTest, TestGroupingSearch, TestFiltering, OverseerCollectionProcessorTest, ShowFileRequestHandlerTest, SolrIndexSplitterTest, AnalysisAfterCoreReloadTest, SuggesterTest, TestTrie, SuggesterWFSTTest, WordBreakSolrSpellCheckerTest, FieldMutatingUpdateProcessorTest, TestRemoteStreaming, TestSurroundQueryParser, TestQueryUtils, StandardRequestHandlerTest, TestWriterPerf, DocumentBuilderTest, TestSolrQueryParser, SolrPluginUtilsTest, TestJmxIntegration, JSONWriterTest, MBeansHandlerTest, JsonLoaderTest, CSVRequestHandlerTest, HighlighterConfigTest, TestQuerySenderListener, TestQuerySenderNoQuery, ResponseLogComponentTest, TestMergePolicyConfig, TestSolrDeletionPolicy2, MultiTermTest, TestBinaryField, TestConfig, ExternalFileFieldSortTest, TestLuceneMatchVersion, TestCodecSupport, SynonymTokenizerTest, TestPerFieldSimilarity, TestIBSimilarityFactory, ChaosMonkeyNothingIsSafeTest, LeaderInitiatedRecoveryOnCommitTest, ReplicationFactorTest]
   [junit4] Completed on J1 in 79.52s, 1 test, 1 error <<< FAILURES!

[...truncated 333 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 57 ignored (8 assumptions)

Total time: 158 minutes 57 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 872334198 bytes
Compression is 0.0%
Took 2 min 31 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 807 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/807/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:10994/b/od/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:10994/b/od/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([CBEE9F8476EBC3B5:43BAA05ED817AE4D]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:10994 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28366/dwq/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28366/dwq/collection1
	at __randomizedtesting.SeedInfo.seed([CBEE9F8476EBC3B5:43BAA05ED817AE4D]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28366/dwq/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:28366 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:27433/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:27433/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([CBEE9F8476EBC3B5:43BAA05ED817AE4D]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:27433/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:27433 failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 10070 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/init-core-data-001
   [junit4]   2> 29602 T30 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /dwq/
   [junit4]   2> 29666 T30 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 29678 T31 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 29980 T30 oasc.ZkTestServer.run start zk server on port:47016
   [junit4]   2> 30136 T30 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 30408 T30 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 30618 T38 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@218dc1b2 name:ZooKeeperConnection Watcher:127.0.0.1:47016 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 30619 T30 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 30629 T30 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 30642 T30 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 30787 T30 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 30788 T30 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 30791 T41 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@20f06938 name:ZooKeeperConnection Watcher:127.0.0.1:47016/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 30791 T30 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 30793 T30 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 30814 T30 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 30819 T30 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 30823 T30 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 30826 T30 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 30831 T30 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 30880 T30 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 30888 T30 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 30890 T30 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 30897 T30 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 30898 T30 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 30905 T30 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 30906 T30 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 30913 T30 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 30914 T30 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 30921 T30 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 30922 T30 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 30927 T30 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 30927 T30 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 30932 T30 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 30932 T30 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 30937 T30 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 30938 T30 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 30942 T30 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 30943 T30 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 30948 T30 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 30949 T30 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 32698 T30 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/control-001/cores/collection1
   [junit4]   2> 32706 T30 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 32711 T30 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:24368
   [junit4]   2> 32712 T30 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/dwq, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/control-001/cores, hostPort=49688, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/tempDir-001/control/data}
   [junit4]   2> 32714 T30 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 32715 T30 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/control-001/'
   [junit4]   2> 32810 T30 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/control-001/solr.xml
   [junit4]   2> 32921 T30 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/control-001/cores
   [junit4]   2> 32922 T30 oasc.CoreContainer.<init> New CoreContainer 2070538716
   [junit4]   2> 32923 T30 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/control-001/]
   [junit4]   2> 32924 T30 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/control-001/lib
   [junit4]   2> 32926 T30 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/control-001/lib).
   [junit4]   2> 32950 T30 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 32951 T30 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 32952 T30 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 32953 T30 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 32954 T30 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 32954 T30 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 32955 T30 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 32956 T30 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 32957 T30 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 32958 T30 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 32958 T30 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 32970 T30 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 32973 T30 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 32973 T30 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 32974 T30 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 32975 T30 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:47016/solr
   [junit4]   2> 33066 T30 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 33067 T30 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 33068 T30 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 33074 T53 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@74cff77f name:ZooKeeperConnection Watcher:127.0.0.1:47016 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 33075 T30 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 33076 T30 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 33116 T30 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 33119 T56 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1849bb18 name:ZooKeeperConnection Watcher:127.0.0.1:47016/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 33120 T30 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 33152 T30 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 33178 T30 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 33193 T30 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 33198 T30 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 33204 T30 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 33250 T30 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 33255 T30 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 33259 T30 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 33263 T30 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49688_dwq
   [junit4]   2> 33277 T30 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49688_dwq
   [junit4]   2> 33283 T30 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 33287 T30 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 33291 T30 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 33331 T30 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:49688_dwq
   [junit4]   2> 33332 T30 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 33336 T30 oasc.Overseer.start Overseer (id=93595920313745411-127.0.0.1:49688_dwq-n_0000000000) starting
   [junit4]   2> 33371 T30 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 33428 T30 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 34117 T58 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 34117 T30 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 34162 T57 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 34166 T30 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/control-001/cores
   [junit4]   2> 34198 T30 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, name=collection1, dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/control-001/cores/collection1, loadOnStartup=true, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/control-001/cores/collection1/, config=solrconfig.xml, shard=, schema=schema.xml, collection=control_collection}
   [junit4]   2> 34199 T30 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/control-001/cores/collection1/
   [junit4]   2> 34199 T30 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 34202 T60 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 34203 T60 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 34231 T56 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 34231 T60 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 34240 T57 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49688/dwq",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:49688_dwq",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"} current state version: 0
   [junit4]   2> 34313 T57 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49688/dwq",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:49688_dwq",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 34323 T57 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 34390 T57 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 34415 T56 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 35233 T60 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 35233 T60 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 35235 T60 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 35236 T60 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 35239 T60 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 35240 T60 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/control-001/cores/collection1/'
   [junit4]   2> 35309 T60 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 35342 T60 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 35385 T60 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 35425 T60 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 35494 T60 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 35496 T60 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 35579 T60 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 36671 T60 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 36720 T60 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 36724 T60 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 36846 T60 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 36853 T60 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 36862 T60 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 36877 T60 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 36878 T60 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 36880 T60 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 36883 T60 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 36884 T60 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 36939 T60 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 36940 T60 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 36941 T60 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/control-001/cores/collection1/data/
   [junit4]   2> 36942 T60 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@28c61629
   [junit4]   2> 37149 T60 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/control-001/cores/collection1/data
   [junit4]   2> 37150 T60 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/control-001/cores/collection1/data/index/
   [junit4]   2> 37150 T60 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 37151 T60 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/control-001/cores/collection1/data/index
   [junit4]   2> 37152 T60 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1467724258, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 37171 T60 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@60e67db4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3c46e61f),segFN=segments_1,generation=1}
   [junit4]   2> 37172 T60 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 37195 T60 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 37301 T60 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 37302 T60 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 37303 T60 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 37304 T60 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 37304 T60 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 37305 T60 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 37306 T60 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 37307 T60 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 37334 T60 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 37350 T60 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 37354 T60 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 37359 T60 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 37382 T60 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /schema,/admin/plugins,/admin/system,/admin/logging,/update/json,/admin/segments,/admin/file,/get,/replication,/update/json/docs,/admin/properties,/admin/luke,/admin/threads,/admin/ping,/admin/mbeans,/config,/update/csv,/update,standard
   [junit4]   2> 37393 T60 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 37396 T60 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 37396 T60 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 37399 T60 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 37399 T60 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 37402 T60 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=37, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 37404 T60 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@60e67db4 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@3c46e61f),segFN=segments_1,generation=1}
   [junit4]   2> 37404 T60 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 37405 T60 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@66d9ce6e[collection1] main
   [junit4]   2> 37406 T60 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 37409 T60 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 37409 T60 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 37426 T60 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 37427 T60 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 37427 T60 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 37429 T60 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 37430 T60 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 37430 T60 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 37431 T60 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 37435 T61 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@66d9ce6e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 37444 T60 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 37446 T64 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:49688/dwq collection:control_collection shard:shard1
   [junit4]   2> 37447 T30 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 37448 T30 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 37492 T64 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 37506 T64 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 37510 T56 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 37511 T64 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 37512 T64 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 37513 T57 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C1 name=collection1 org.apache.solr.core.SolrCore@720fb349 url=http://127.0.0.1:49688/dwq/collection1 node=127.0.0.1:49688_dwq C1_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:49688/dwq, state=down, core=collection1, node_name=127.0.0.1:49688_dwq}
   [junit4]   2> 37523 T64 C:control_collection S:shard1 C1 P49688 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:49688/dwq/collection1/
   [junit4]   2> 37524 T64 C:control_collection S:shard1 C1 P49688 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 37525 T64 C:control_collection S:shard1 C1 P49688 oasc.SyncStrategy.syncToMe http://127.0.0.1:49688/dwq/collection1/ has no replicas
   [junit4]   2> 37525 T64 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:49688/dwq/collection1/ shard1
   [junit4]   2> 37530 T56 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 37543 T64 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 37552 T56 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 37555 T57 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49688/dwq",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 37630 T30 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 37632 T30 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 37634 T68 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@523a44ea name:ZooKeeperConnection Watcher:127.0.0.1:47016/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 37635 T30 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 37636 T30 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 37637 T30 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 37648 T30 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 37648 T30 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 37649 T30 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 37650 T30 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 37653 T71 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2c12e0e name:ZooKeeperConnection Watcher:127.0.0.1:47016/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 37653 T30 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 37655 T30 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 37659 T56 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 37661 T57 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 37662 T57 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 37664 T68 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 37664 T56 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 37708 T64 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:49688/dwq/collection1/ and leader is http://127.0.0.1:49688/dwq/collection1/
   [junit4]   2> 37708 T64 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:49688/dwq
   [junit4]   2> 37709 T64 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 37709 T64 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 37710 T64 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 37714 T56 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 37716 T57 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49688/dwq",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:49688_dwq",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"} current state version: 3
   [junit4]   2> 37718 T57 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49688/dwq",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:49688_dwq",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 37720 T57 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 37723 T56 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 37723 T68 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 37837 T56 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 37837 T68 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 40065 T30 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/shard-1-001/cores/collection1
   [junit4]   2> 40070 T30 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/shard-1-001
   [junit4]   2> 40071 T30 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 40075 T30 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:28522
   [junit4]   2> 40076 T30 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=28366, solrconfig=solrconfig.xml, hostContext=/dwq, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/shard-1-001/cores}
   [junit4]   2> 40078 T30 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 40079 T30 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/shard-1-001/'
   [junit4]   2> 40158 T30 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/shard-1-001/solr.xml
   [junit4]   2> 40261 T30 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/shard-1-001/cores
   [junit4]   2> 40263 T30 oasc.CoreContainer.<init> New CoreContainer 262453308
   [junit4]   2> 40264 T30 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/shard-1-001/]
   [junit4]   2> 40265 T30 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/shard-1-001/lib
   [junit4]   2> 40267 T30 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/shard-1-001/lib).
   [junit4]   2> 40293 T30 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 40294 T30 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 40295 T30 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 40296 T30 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 40297 T30 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 40298 T30 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 40299 T30 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 40300 T30 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 40301 T30 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 40302 T30 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 40303 T30 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 40316 T30 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 40319 T30 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 40320 T30 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 40321 T30 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 40322 T30 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:47016/solr
   [junit4]   2> 40322 T30 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 40323 T30 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 40325 T30 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 40329 T83 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7f4c4c6 name:ZooKeeperConnection Watcher:127.0.0.1:47016 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 40330 T30 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 40331 T30 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 40337 T30 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 40341 T86 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2ef69561 name:ZooKeeperConnection Watcher:127.0.0.1:47016/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 40342 T30 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 40357 T30 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 41377 T30 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:28366_dwq
   [junit4]   2> 41379 T30 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:28366_dwq
   [junit4]   2> 41389 T30 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 41408 T30 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/shard-1-001/cores
   [junit4]   2> 41413 T30 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, dataDir=data/, config=solrconfig.xml, shard=, name=collection1, collection=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/shard-1-001/cores/collection1/, coreNodeName=, schema=schema.xml, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/shard-1-001/cores/collection1}
   [junit4]   2> 41415 T30 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/shard-1-001/cores/collection1/
   [junit4]   2> 41416 T30 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 41425 T87 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 41426 T87 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 41431 T56 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 41437 T87 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 41437 T57 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28366/dwq",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:28366_dwq",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"} current state version: 5
   [junit4]   2> 41438 T87 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 41440 T87 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 41441 T57 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28366/dwq",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:28366_dwq",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1"}
   [junit4]   2> 41443 T57 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 41444 T57 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 41456 T87 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 41456 T57 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 41457 T87 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 41461 T86 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 41464 T86 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 42459 T87 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 42460 T87 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 42463 T87 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 42463 T87 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 42466 T87 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 42467 T87 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/shard-1-001/cores/collection1/'
   [junit4]   2> 42503 T87 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 42511 T87 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 42556 T87 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 42608 T87 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 42749 T87 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 42752 T87 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 42858 T87 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 43563 T87 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 43603 T87 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 43608 T87 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 43675 T87 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 43688 T87 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 43701 T87 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 43705 T87 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 43705 T87 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 43706 T87 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 43709 T87 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 43710 T87 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 43711 T87 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 43712 T87 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 43713 T87 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 43713 T87 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@28c61629
   [junit4]   2> 43716 T87 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/shard-1-001/cores/collection1/data
   [junit4]   2> 43717 T87 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 43717 T87 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 43718 T87 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CBEE9F8476EBC3B5-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 43719 T87 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1467724258, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 43722 T87 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4f9d6258 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@65a3a83d),segFN=segments_1,generation=1}
   [junit4]   2> 43723 T87 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 43746 T87 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 43748 T87 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 43749 T87 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 43749 T87 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 43750 T87 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 43751 T87 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 43752 T87 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 43754 T87 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 43754 T87 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 43788 T87 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 43796 T87 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 43803 T87 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 43809 T87 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 43865 T87 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /schema,/admin/plugins,/admin/system,/admin/logging,/update/json,/admin/segments,/admin/file,/get,/replication,/update/json/docs,/admin/properties,/admin/luke,/admin/threads,/admin/ping,/admin/mbeans,/config,/update/csv,/update,standard
   [junit4]   2> 43869 T87 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 43876 T87 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 43876 T87 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 43879 T87 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 43879 T87 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 43883 T87 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=37, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 43885 T87 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4f9d6258 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@65a3a83d),segFN=segments_1,generation=1}
   [junit4]   2> 43885 T87 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 43886 T87 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4242dfb[collection1] main
   [junit4]   2> 43887 T87 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 43889 T87 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 43890 T87 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 43892 T87 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 43893 T87 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 43894 T87 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 43896 T87 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 43897 T87 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 43898 T87 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 43898 T87 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 43905 T88 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4242dfb[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 43907 T87 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 43909 T91 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:28366/dwq collection:collection1 shard:shard1
   [junit4]   2> 43910 T30 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 43912 T30 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 43912 T91 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 43927 T91 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 43932 T56 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 43933 T91 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 43933 T91 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C2 name=collection1 org.apache.solr.core.SolrCore@57f74a57 url=http://127.0.0.1:28366/dwq/collection1 node=127.0.0.1:28366_dwq C2_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:28366/dwq, state=down, core=collection1, node_name=127.0.0.1:28366_dwq}
   [junit4]   2> 43934 T91 C:collection1 S:shard1 C2 P28366 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:28366/dwq/collection1/
   [junit4]   2> 43935 T57 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 43936 T91 C:collection1 S:shard1 C2 P28366 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 43937 T91 C:collection1 S:shard1 C2 P28366 oasc.SyncStrategy.syncToMe http://127.0.0.1:28366/dwq/collection1/ has no replicas
   [junit4]   2> 43937 T91 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:28366/dwq/collection1/ shard1
   [junit4]   2> 43938 T91 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 43939 T57 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 43942 T86 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 43945 T86 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 43954 T57 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28366/dwq",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 43958 T57 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 43961 T86 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 43964 T86 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 43969 T56 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 44003 T91 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:28366/dwq/collection1/ and leader is http://127.0.0.1:28366/dwq/collection1/
   [junit4]   2> 44004 T91 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:28366/dwq
   [junit4]   2> 44005 T91 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 44006 T91 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 44006 T91 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 44011 T56 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 44014 T57 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:28366/dwq",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:28366_dwq",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":

[...truncated too long message...]

  [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:27433/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: 127.0.0.1:27433 failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:882)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 53 more
   [junit4]   2> 2345641 T10123 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.ReplicationFactorTest CBEE9F8476EBC3B5-001
   [junit4]   2> 76024 T10122 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {a_t=PostingsFormat(name=LuceneVarGapDocFreqInterval), id=PostingsFormat(name=Memory doPackFST= false), multiDefault=PostingsFormat(name=LuceneVarGapDocFreqInterval), range_facet_l=PostingsFormat(name=LuceneVarGapDocFreqInterval), intDefault=Lucene50(blocksize=128), _version_=Lucene50(blocksize=128), timestamp=PostingsFormat(name=LuceneVarGapDocFreqInterval), text=PostingsFormat(name=Asserting)}, docValues:{timestamp=DocValuesFormat(name=Asserting)}, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=zh_SG, timezone=America/Indiana/Tell_City
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=159301280,total=372768768
   [junit4]   2> NOTE: All tests run in this JVM: [TestDefaultSearchFieldResource, TestShardHandlerFactory, RecoveryAfterSoftCommitTest, TestSystemIdResolver, CSVRequestHandlerTest, DocValuesMultiTest, DistributedFacetPivotLongTailTest, TestRandomMergePolicy, TestReplicationHandlerBackup, ScriptEngineTest, SolrCoreTest, DistributedQueueTest, TestManagedSynonymFilterFactory, OutputWriterTest, AssignTest, TestMaxScoreQueryParser, BadIndexSchemaTest, UnloadDistributedZkTest, SampleTest, TestBulkSchemaConcurrent, TestDefaultSimilarityFactory, TestBlendedInfixSuggestions, DistributedMLTComponentTest, TestConfigReload, TermsComponentTest, HdfsChaosMonkeySafeLeaderTest, TestRebalanceLeaders, TestNonDefinedSimilarityFactory, ResponseLogComponentTest, TestFiltering, AlternateDirectoryTest, CurrencyFieldXmlFileTest, CoreAdminHandlerTest, TestRangeQuery, TestSchemaManager, LeaderElectionTest, CoreAdminRequestStatusTest, DistributedIntervalFacetingTest, DeleteLastCustomShardedReplicaTest, TestSearcherReuse, UpdateRequestProcessorFactoryTest, XsltUpdateRequestHandlerTest, TestIndexingPerformance, TestQuerySenderListener, TestJmxMonitoredMap, SSLMigrationTest, ClusterStateTest, NumericFieldsTest, TestCSVLoader, TestJmxIntegration, TestArbitraryIndexDir, TestSerializedLuceneMatchVersion, MergeStrategyTest, TestFieldTypeResource, SpellPossibilityIteratorTest, TestClassNameShortening, SpatialHeatmapFacetsTest, TestComplexPhraseQParserPlugin, BasicDistributedZkTest, ChaosMonkeySafeLeaderTest, SyncSliceTest, TestFaceting, TestHashPartitioner, DistributedSpellCheckComponentTest, ZkControllerTest, TestJoin, TestSolr4Spatial, SpellCheckComponentTest, TestGroupingSearch, BasicFunctionalityTest, TestIndexSearcher, CurrencyFieldOpenExchangeTest, SolrIndexSplitterTest, SimplePostToolTest, TestCoreDiscovery, SignatureUpdateProcessorFactoryTest, SpellCheckCollatorTest, DocValuesTest, SuggesterTSTTest, SuggesterWFSTTest, TestUpdate, TestWordDelimiterFilterFactory, QueryEqualityTest, SortByFunctionTest, SolrInfoMBeanTest, LukeRequestHandlerTest, DirectSolrSpellCheckerTest, DocumentAnalysisRequestHandlerTest, PrimitiveFieldTypeTest, TestValueSourceCache, MoreLikeThisHandlerTest, FieldAnalysisRequestHandlerTest, RegexBoostProcessorTest, IndexSchemaTest, TestAnalyzedSuggestions, BinaryUpdateRequestHandlerTest, TestBinaryResponseWriter, TestQuerySenderNoQuery, SolrIndexConfigTest, MultiTermTest, TestBinaryField, TestConfig, TestPostingsSolrHighlighter, DirectSolrConnectionTest, NotRequiredUniqueKeyTest, TestCharFilters, TestCodecSupport, SynonymTokenizerTest, TestXIncludeConfig, EchoParamsTest, TestDFRSimilarityFactory, TestPerFieldSimilarity, TestLMDirichletSimilarityFactory, TestIBSimilarityFactory, TestBM25SimilarityFactory, TimeZoneUtilsTest, OpenExchangeRatesOrgProviderTest, TestFastLRUCache, DateMathParserTest, TestSuggestSpellingConverter, DateFieldTest, SpellingQueryConverterTest, RAMDirectoryFactoryTest, TestLRUCache, TestUtils, TestDocumentBuilder, ZkNodePropsTest, SliceStateTest, SystemInfoHandlerTest, UUIDFieldTest, FileUtilsTest, TestRTGBase, CursorPagingTest, TestCursorMarkWithoutUniqueKey, TestDistributedMissingSort, TestTolerantSearch, TestEmbeddedSolrServerConstructors, TestJettySolrRunner, AliasIntegrationTest, CloudExitableDirectoryReaderTest, DeleteReplicaTest, DistribDocExpirationUpdateProcessorTest, ExternalCollectionsTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverseerStatusTest, ReplicationFactorTest]
   [junit4] Completed on J3 in 76.30s, 1 test, 1 error <<< FAILURES!

[...truncated 340 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 57 ignored (8 assumptions)

Total time: 151 minutes 44 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 875602529 bytes
Compression is 0.0%
Took 2 min 41 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 806 - Still Failing

Posted by Areek Zillur <ar...@gmail.com>.
I committed a fix for this.
-Areek

On Fri, Apr 3, 2015 at 11:52 AM, Apache Jenkins Server <
jenkins@builds.apache.org> wrote:

> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/806/
>
> 1 tests failed.
> REGRESSION:
> org.apache.lucene.search.suggest.document.SuggestFieldTest.testDupSuggestFieldValues
>
> Error Message:
> MockDirectoryWrapper: cannot close: there are still open files:
> {_ak_completion_0.lkp=1, _aj_completion_0.lkp=1, _ak_completion_0.tim=1,
> _ak_completion_0.doc=1, _ak.nvd=1, _aj_completion_0.pay=1,
> _aj_completion_0.pos=1, _ak_completion_0.pos=1, _ak.fdt=1, _aj.nvd=1,
> _aj_completion_0.tim=1, _aj_completion_0.doc=1, _ak_completion_0.pay=1,
> _aj.fdt=1}
>
> Stack Trace:
> java.lang.RuntimeException: MockDirectoryWrapper: cannot close: there are
> still open files: {_ak_completion_0.lkp=1, _aj_completion_0.lkp=1,
> _ak_completion_0.tim=1, _ak_completion_0.doc=1, _ak.nvd=1,
> _aj_completion_0.pay=1, _aj_completion_0.pos=1, _ak_completion_0.pos=1,
> _ak.fdt=1, _aj.nvd=1, _aj_completion_0.tim=1, _aj_completion_0.doc=1,
> _ak_completion_0.pay=1, _aj.fdt=1}
>         at
> org.apache.lucene.store.MockDirectoryWrapper.close(MockDirectoryWrapper.java:747)
>         at
> org.apache.lucene.search.suggest.document.SuggestFieldTest.after(SuggestFieldTest.java:80)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:894)
>         at
> org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at
> org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at
> org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at
> org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at
> org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
>         at
> org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at
> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at
> com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at
> org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at
> org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at
> org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at
> com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at
> com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
> Caused by: java.lang.RuntimeException: unclosed IndexInput:
> _aj_completion_0.pos
>         at
> org.apache.lucene.store.MockDirectoryWrapper.addFileHandle(MockDirectoryWrapper.java:622)
>         at
> org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:666)
>         at
> org.apache.lucene.codecs.lucene50.Lucene50PostingsReader.<init>(Lucene50PostingsReader.java:92)
>         at
> org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.fieldsProducer(Lucene50PostingsFormat.java:443)
>         at
> org.apache.lucene.search.suggest.document.CompletionFieldsProducer.<init>(CompletionFieldsProducer.java:93)
>         at
> org.apache.lucene.search.suggest.document.CompletionPostingsFormat.fieldsProducer(CompletionPostingsFormat.java:119)
>         at
> org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:258)
>         at
> org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:336)
>         at
> org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:104)
>         at
> org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:65)
>         at
> org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:132)
>         at
> org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:184)
>         at
> org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:99)
>         at
> org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:430)
>         at
> org.apache.lucene.index.RandomIndexWriter.getReader(RandomIndexWriter.java:352)
>         at
> org.apache.lucene.index.RandomIndexWriter.getReader(RandomIndexWriter.java:280)
>         at
> org.apache.lucene.search.suggest.document.SuggestFieldTest.testDupSuggestFieldValues(SuggestFieldTest.java:162)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
>         at
> com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
>         ... 28 more
>
>
>
>
> Build Log:
> [...truncated 8852 lines...]
>    [junit4] Suite:
> org.apache.lucene.search.suggest.document.SuggestFieldTest
>    [junit4]   2> NOTE: download the large Jenkins line-docs file by
> running 'ant get-jenkins-line-docs' in the lucene directory.
>    [junit4]   2> NOTE: reproduce with: ant test
> -Dtestcase=SuggestFieldTest -Dtests.method=testDupSuggestFieldValues
> -Dtests.seed=21CC108D79B60A32 -Dtests.multiplier=2 -Dtests.nightly=true
> -Dtests.slow=true
> -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt
> -Dtests.locale=bg_BG -Dtests.timezone=Asia/Colombo -Dtests.asserts=true
> -Dtests.file.encoding=US-ASCII
>    [junit4] ERROR   8.26s J3 | SuggestFieldTest.testDupSuggestFieldValues
> <<<
>    [junit4]    > Throwable #1: java.lang.AssertionError:
>    [junit4]    > Expected: <1736>
>    [junit4]    >      got: <1362>
>    [junit4]    >        at
> org.apache.lucene.search.suggest.document.SuggestFieldTest.assertSuggestions(SuggestFieldTest.java:740)
>    [junit4]    >        at
> org.apache.lucene.search.suggest.document.SuggestFieldTest.testDupSuggestFieldValues(SuggestFieldTest.java:171)
>    [junit4]    >        at java.lang.Thread.run(Thread.java:745)Throwable
> #2: java.lang.RuntimeException: MockDirectoryWrapper: cannot close: there
> are still open files: {_ak_completion_0.lkp=1, _aj_completion_0.lkp=1,
> _ak_completion_0.tim=1, _ak_completion_0.doc=1, _ak.nvd=1,
> _aj_completion_0.pay=1, _aj_completion_0.pos=1, _ak_completion_0.pos=1,
> _ak.fdt=1, _aj.nvd=1, _aj_completion_0.tim=1, _aj_completion_0.doc=1,
> _ak_completion_0.pay=1, _aj.fdt=1}
>    [junit4]    >        at
> org.apache.lucene.store.MockDirectoryWrapper.close(MockDirectoryWrapper.java:747)
>    [junit4]    >        at
> org.apache.lucene.search.suggest.document.SuggestFieldTest.after(SuggestFieldTest.java:80)
>    [junit4]    >        at java.lang.Thread.run(Thread.java:745)
>    [junit4]    > Caused by: java.lang.RuntimeException: unclosed
> IndexInput: _aj_completion_0.pos
>    [junit4]    >        at
> org.apache.lucene.store.MockDirectoryWrapper.addFileHandle(MockDirectoryWrapper.java:622)
>    [junit4]    >        at
> org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:666)
>    [junit4]    >        at
> org.apache.lucene.codecs.lucene50.Lucene50PostingsReader.<init>(Lucene50PostingsReader.java:92)
>    [junit4]    >        at
> org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.fieldsProducer(Lucene50PostingsFormat.java:443)
>    [junit4]    >        at
> org.apache.lucene.search.suggest.document.CompletionFieldsProducer.<init>(CompletionFieldsProducer.java:93)
>    [junit4]    >        at
> org.apache.lucene.search.suggest.document.CompletionPostingsFormat.fieldsProducer(CompletionPostingsFormat.java:119)
>    [junit4]    >        at
> org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:258)
>    [junit4]    >        at
> org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:336)
>    [junit4]    >        at
> org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:104)
>    [junit4]    >        at
> org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:65)
>    [junit4]    >        at
> org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:132)
>    [junit4]    >        at
> org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:184)
>    [junit4]    >        at
> org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:99)
>    [junit4]    >        at
> org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:430)
>    [junit4]    >        at
> org.apache.lucene.index.RandomIndexWriter.getReader(RandomIndexWriter.java:352)
>    [junit4]    >        at
> org.apache.lucene.index.RandomIndexWriter.getReader(RandomIndexWriter.java:280)
>    [junit4]    >        at
> org.apache.lucene.search.suggest.document.SuggestFieldTest.testDupSuggestFieldValues(SuggestFieldTest.java:162)
>    [junit4]    >        ... 28 more
>    [junit4]   2> NOTE: leaving temporary files on disk at:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/suggest/test/J3/temp/lucene.search.suggest.document.SuggestFieldTest
> 21CC108D79B60A32-001
>    [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {},
> docValues:{}, sim=DefaultSimilarity, locale=bg_BG, timezone=Asia/Colombo
>    [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation
> 1.7.0_65 (64-bit)/cpus=16,threads=1,free=199522176,total=373293056
>    [junit4]   2> NOTE: All tests run in this JVM: [TestDirectSpellChecker,
> TestWordBreakSpellChecker, BytesRefSortersTest, BlendedInfixSuggesterTest,
> FSTCompletionTest, PersistenceTest, TestLevenshteinDistance,
> TestHighFrequencyDictionary, TestJaroWinklerDistance, TestNGramDistance,
> TestInputIterator, TestFreeTextSuggester, TestSuggestStopFilter,
> SuggestFieldTest]
>    [junit4] Completed on J3 in 41.18s, 16 tests, 1 error <<< FAILURES!
>
> [...truncated 12 lines...]
> BUILD FAILED
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536:
> The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477:
> The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61:
> The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39:
> The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:456:
> The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:2166:
> The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/module-build.xml:58:
> The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370:
> The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976:
> There were test failures: 26 suites, 162 tests, 1 error, 1 ignored
>
> Total time: 107 minutes 56 seconds
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
> Archived 5 artifacts
> Archive block size is 32768
> Received 0 blocks and 240072 bytes
> Compression is 0.0%
> Took 0.43 sec
> Recording test results
> Email was triggered for: Failure
> Sending email for trigger: Failure
>
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>

[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 806 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/806/

1 tests failed.
REGRESSION:  org.apache.lucene.search.suggest.document.SuggestFieldTest.testDupSuggestFieldValues

Error Message:
MockDirectoryWrapper: cannot close: there are still open files: {_ak_completion_0.lkp=1, _aj_completion_0.lkp=1, _ak_completion_0.tim=1, _ak_completion_0.doc=1, _ak.nvd=1, _aj_completion_0.pay=1, _aj_completion_0.pos=1, _ak_completion_0.pos=1, _ak.fdt=1, _aj.nvd=1, _aj_completion_0.tim=1, _aj_completion_0.doc=1, _ak_completion_0.pay=1, _aj.fdt=1}

Stack Trace:
java.lang.RuntimeException: MockDirectoryWrapper: cannot close: there are still open files: {_ak_completion_0.lkp=1, _aj_completion_0.lkp=1, _ak_completion_0.tim=1, _ak_completion_0.doc=1, _ak.nvd=1, _aj_completion_0.pay=1, _aj_completion_0.pos=1, _ak_completion_0.pos=1, _ak.fdt=1, _aj.nvd=1, _aj_completion_0.tim=1, _aj_completion_0.doc=1, _ak_completion_0.pay=1, _aj.fdt=1}
	at org.apache.lucene.store.MockDirectoryWrapper.close(MockDirectoryWrapper.java:747)
	at org.apache.lucene.search.suggest.document.SuggestFieldTest.after(SuggestFieldTest.java:80)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:894)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.RuntimeException: unclosed IndexInput: _aj_completion_0.pos
	at org.apache.lucene.store.MockDirectoryWrapper.addFileHandle(MockDirectoryWrapper.java:622)
	at org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:666)
	at org.apache.lucene.codecs.lucene50.Lucene50PostingsReader.<init>(Lucene50PostingsReader.java:92)
	at org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.fieldsProducer(Lucene50PostingsFormat.java:443)
	at org.apache.lucene.search.suggest.document.CompletionFieldsProducer.<init>(CompletionFieldsProducer.java:93)
	at org.apache.lucene.search.suggest.document.CompletionPostingsFormat.fieldsProducer(CompletionPostingsFormat.java:119)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:258)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:336)
	at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:104)
	at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:65)
	at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:132)
	at org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:184)
	at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:99)
	at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:430)
	at org.apache.lucene.index.RandomIndexWriter.getReader(RandomIndexWriter.java:352)
	at org.apache.lucene.index.RandomIndexWriter.getReader(RandomIndexWriter.java:280)
	at org.apache.lucene.search.suggest.document.SuggestFieldTest.testDupSuggestFieldValues(SuggestFieldTest.java:162)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	... 28 more




Build Log:
[...truncated 8852 lines...]
   [junit4] Suite: org.apache.lucene.search.suggest.document.SuggestFieldTest
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SuggestFieldTest -Dtests.method=testDupSuggestFieldValues -Dtests.seed=21CC108D79B60A32 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=bg_BG -Dtests.timezone=Asia/Colombo -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   8.26s J3 | SuggestFieldTest.testDupSuggestFieldValues <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: 
   [junit4]    > Expected: <1736>
   [junit4]    >      got: <1362>
   [junit4]    > 	at org.apache.lucene.search.suggest.document.SuggestFieldTest.assertSuggestions(SuggestFieldTest.java:740)
   [junit4]    > 	at org.apache.lucene.search.suggest.document.SuggestFieldTest.testDupSuggestFieldValues(SuggestFieldTest.java:171)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)Throwable #2: java.lang.RuntimeException: MockDirectoryWrapper: cannot close: there are still open files: {_ak_completion_0.lkp=1, _aj_completion_0.lkp=1, _ak_completion_0.tim=1, _ak_completion_0.doc=1, _ak.nvd=1, _aj_completion_0.pay=1, _aj_completion_0.pos=1, _ak_completion_0.pos=1, _ak.fdt=1, _aj.nvd=1, _aj_completion_0.tim=1, _aj_completion_0.doc=1, _ak_completion_0.pay=1, _aj.fdt=1}
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.close(MockDirectoryWrapper.java:747)
   [junit4]    > 	at org.apache.lucene.search.suggest.document.SuggestFieldTest.after(SuggestFieldTest.java:80)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: java.lang.RuntimeException: unclosed IndexInput: _aj_completion_0.pos
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.addFileHandle(MockDirectoryWrapper.java:622)
   [junit4]    > 	at org.apache.lucene.store.MockDirectoryWrapper.openInput(MockDirectoryWrapper.java:666)
   [junit4]    > 	at org.apache.lucene.codecs.lucene50.Lucene50PostingsReader.<init>(Lucene50PostingsReader.java:92)
   [junit4]    > 	at org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.fieldsProducer(Lucene50PostingsFormat.java:443)
   [junit4]    > 	at org.apache.lucene.search.suggest.document.CompletionFieldsProducer.<init>(CompletionFieldsProducer.java:93)
   [junit4]    > 	at org.apache.lucene.search.suggest.document.CompletionPostingsFormat.fieldsProducer(CompletionPostingsFormat.java:119)
   [junit4]    > 	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:258)
   [junit4]    > 	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:336)
   [junit4]    > 	at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:104)
   [junit4]    > 	at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:65)
   [junit4]    > 	at org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:132)
   [junit4]    > 	at org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:184)
   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:99)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:430)
   [junit4]    > 	at org.apache.lucene.index.RandomIndexWriter.getReader(RandomIndexWriter.java:352)
   [junit4]    > 	at org.apache.lucene.index.RandomIndexWriter.getReader(RandomIndexWriter.java:280)
   [junit4]    > 	at org.apache.lucene.search.suggest.document.SuggestFieldTest.testDupSuggestFieldValues(SuggestFieldTest.java:162)
   [junit4]    > 	... 28 more
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/suggest/test/J3/temp/lucene.search.suggest.document.SuggestFieldTest 21CC108D79B60A32-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=DefaultSimilarity, locale=bg_BG, timezone=Asia/Colombo
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=199522176,total=373293056
   [junit4]   2> NOTE: All tests run in this JVM: [TestDirectSpellChecker, TestWordBreakSpellChecker, BytesRefSortersTest, BlendedInfixSuggesterTest, FSTCompletionTest, PersistenceTest, TestLevenshteinDistance, TestHighFrequencyDictionary, TestJaroWinklerDistance, TestNGramDistance, TestInputIterator, TestFreeTextSuggester, TestSuggestStopFilter, SuggestFieldTest]
   [junit4] Completed on J3 in 41.18s, 16 tests, 1 error <<< FAILURES!

[...truncated 12 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:456: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:2166: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/module-build.xml:58: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 26 suites, 162 tests, 1 error, 1 ignored

Total time: 107 minutes 56 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 240072 bytes
Compression is 0.0%
Took 0.43 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 805 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/805/

5 tests failed.
REGRESSION:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1427989704495,generation=2,filelist=[_9q.cfe, _9q.cfs, _9q.si, _9u.cfe, _9u.cfs, _9u.si, _9y.cfe, _9y.cfs, _9y.si, _9z.cfe, _9z.cfs, _9z.si, _a0.cfe, _a0.cfs, _a0.si, _a1.cfe, _a1.cfs, _a1.si, segments_2]}]> but was:<[{indexVersion=1427989704495,generation=2,filelist=[_9q.cfe, _9q.cfs, _9q.si, _9u.cfe, _9u.cfs, _9u.si, _9y.cfe, _9y.cfs, _9y.si, _9z.cfe, _9z.cfs, _9z.si, _a0.cfe, _a0.cfs, _a0.si, _a1.cfe, _a1.cfs, _a1.si, segments_2]}, {indexVersion=1427989704495,generation=3,filelist=[_9y.cfe, _9y.cfs, _9y.si, _a1.cfe, _a1.cfs, _a1.si, _a2.cfe, _a2.cfs, _a2.si, segments_3]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1427989704495,generation=2,filelist=[_9q.cfe, _9q.cfs, _9q.si, _9u.cfe, _9u.cfs, _9u.si, _9y.cfe, _9y.cfs, _9y.si, _9z.cfe, _9z.cfs, _9z.si, _a0.cfe, _a0.cfs, _a0.si, _a1.cfe, _a1.cfs, _a1.si, segments_2]}]> but was:<[{indexVersion=1427989704495,generation=2,filelist=[_9q.cfe, _9q.cfs, _9q.si, _9u.cfe, _9u.cfs, _9u.si, _9y.cfe, _9y.cfs, _9y.si, _9z.cfe, _9z.cfs, _9z.si, _a0.cfe, _a0.cfs, _a0.si, _a1.cfe, _a1.cfs, _a1.si, segments_2]}, {indexVersion=1427989704495,generation=3,filelist=[_9y.cfe, _9y.cfs, _9y.si, _a1.cfe, _a1.cfs, _a1.si, _a2.cfe, _a2.cfs, _a2.si, segments_3]}]>
	at __randomizedtesting.SeedInfo.seed([4661B87F5DD5B6B8:63B6A34F2D9DB8BB]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1138)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.handler.component.DistributedFacetPivotSmallTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=3568, name=Thread-1249, state=RUNNABLE, group=TGRP-DistributedFacetPivotSmallTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=3568, name=Thread-1249, state=RUNNABLE, group=TGRP-DistributedFacetPivotSmallTest]
Caused by: org.apache.solr.client.solrj.impl.HttpSolrClient$RemoteSolrException: Error from server at http://127.0.0.1:46675//collection1: org.apache.solr.client.solrj.SolrServerException: No live SolrServers available to handle this request:[http://127.0.0.1:46675//collection1, http://[ff01::083]:33332, http://[ff01::114]:33332, http://[ff01::213]:33332]
	at __randomizedtesting.SeedInfo.seed([4661B87F5DD5B6B8]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:556)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$5.run(BaseDistributedSearchTestCase.java:621)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:46858/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:46858/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([4661B87F5DD5B6B8:CE3587A5F329DB40]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31843/pva/q/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31843/pva/q/collection1
	at __randomizedtesting.SeedInfo.seed([4661B87F5DD5B6B8:CE3587A5F329DB40]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31843/pva/q/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:44860/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:44860/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([4661B87F5DD5B6B8:CE3587A5F329DB40]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:44860/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 10226 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/init-core-data-001
   [junit4]   2> 352438 T1113 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 352443 T1113 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 352443 T1114 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 352543 T1113 oasc.ZkTestServer.run start zk server on port:44840
   [junit4]   2> 352544 T1113 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 352544 T1113 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 352548 T1121 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7d93971d name:ZooKeeperConnection Watcher:127.0.0.1:44840 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 352548 T1113 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 352549 T1113 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 352549 T1113 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 352551 T1113 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 352551 T1113 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 352552 T1124 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@655d1f58 name:ZooKeeperConnection Watcher:127.0.0.1:44840/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 352553 T1113 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 352553 T1113 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 352553 T1113 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 352555 T1113 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 352556 T1113 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 352557 T1113 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 352558 T1113 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 352559 T1113 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 352561 T1113 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 352561 T1113 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 352563 T1113 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 352563 T1113 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 352565 T1113 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 352565 T1113 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 352567 T1113 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 352567 T1113 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 352568 T1113 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 352569 T1113 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 352570 T1113 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 352570 T1113 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 352572 T1113 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 352572 T1113 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 352574 T1113 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 352574 T1113 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 352575 T1113 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 352576 T1113 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 352577 T1113 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 352578 T1113 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 354020 T1113 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/control-001/cores/collection1
   [junit4]   2> 354024 T1113 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 354026 T1113 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:44846
   [junit4]   2> 354027 T1113 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, hostPort=44845, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/tempDir-001/control/data}
   [junit4]   2> 354027 T1113 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3972aa3f
   [junit4]   2> 354027 T1113 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/control-001/'
   [junit4]   2> 354063 T1113 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/control-001/solr.xml
   [junit4]   2> 354106 T1113 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/control-001/cores
   [junit4]   2> 354106 T1113 oasc.CoreContainer.<init> New CoreContainer 693872746
   [junit4]   2> 354106 T1113 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/control-001/]
   [junit4]   2> 354106 T1113 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/control-001/lib
   [junit4]   2> 354107 T1113 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/control-001/lib).
   [junit4]   2> 354116 T1113 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 354117 T1113 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 354117 T1113 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 354117 T1113 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 354117 T1113 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 354117 T1113 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 354118 T1113 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 354118 T1113 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 354118 T1113 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 354118 T1113 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 354118 T1113 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 354119 T1113 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 354119 T1113 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 354119 T1113 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 354119 T1113 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 354120 T1113 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:44840/solr
   [junit4]   2> 354120 T1113 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 354120 T1113 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 354121 T1113 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 354122 T1136 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@28b07d3c name:ZooKeeperConnection Watcher:127.0.0.1:44840 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 354123 T1113 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 354123 T1113 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 354125 T1113 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 354126 T1139 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@79f769e name:ZooKeeperConnection Watcher:127.0.0.1:44840/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 354126 T1113 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 354127 T1113 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 354129 T1113 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 354130 T1113 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 354131 T1113 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 354133 T1113 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 354134 T1113 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 354135 T1113 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 354136 T1113 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 354137 T1113 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44845_
   [junit4]   2> 354137 T1113 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:44845_
   [junit4]   2> 354138 T1113 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 354139 T1113 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 354140 T1113 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 354141 T1113 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:44845_
   [junit4]   2> 354141 T1113 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 354142 T1113 oasc.Overseer.start Overseer (id=93584654516224003-127.0.0.1:44845_-n_0000000000) starting
   [junit4]   2> 354143 T1113 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 354147 T1113 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 354148 T1141 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 354148 T1113 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 354148 T1140 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 354151 T1113 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/control-001/cores
   [junit4]   2> 354153 T1113 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/control-001/cores/collection1/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/control-001/cores/collection1, coreNodeName=, transient=false, config=solrconfig.xml, shard=, dataDir=data/, collection=control_collection, schema=schema.xml}
   [junit4]   2> 354153 T1113 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/control-001/cores/collection1/
   [junit4]   2> 354153 T1113 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 354154 T1143 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 354154 T1143 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 354155 T1143 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 354155 T1139 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 354156 T1140 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:44845",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:44845_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 354156 T1140 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:44845",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:44845_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 354156 T1140 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 354156 T1140 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 354157 T1139 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 355155 T1143 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 355156 T1143 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 355156 T1143 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 355156 T1143 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 355157 T1143 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 355157 T1143 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/control-001/cores/collection1/'
   [junit4]   2> 355170 T1143 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 355173 T1143 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 355181 T1143 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 355197 T1143 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 355232 T1143 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 355234 T1143 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 355241 T1143 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 355584 T1143 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 355597 T1143 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 355599 T1143 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 355608 T1143 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 355612 T1143 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 355615 T1143 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 355616 T1143 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 355616 T1143 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 355616 T1143 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 355617 T1143 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 355617 T1143 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 355617 T1143 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 355617 T1143 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 355617 T1143 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/control-001/cores/collection1/data/
   [junit4]   2> 355618 T1143 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6354743
   [junit4]   2> 355618 T1143 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/control-001/cores/collection1/data
   [junit4]   2> 355618 T1143 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/control-001/cores/collection1/data/index/
   [junit4]   2> 355619 T1143 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 355619 T1143 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/control-001/cores/collection1/data/index
   [junit4]   2> 355619 T1143 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=96.6904296875, floorSegmentMB=1.7919921875, forceMergeDeletesPctAllowed=1.9750866148602197, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 355620 T1143 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@45fe99f3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4e2d446b; maxCacheMB=0.4970703125 maxMergeSizeMB=0.5986328125)),segFN=segments_1,generation=1}
   [junit4]   2> 355620 T1143 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 355624 T1143 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 355625 T1143 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 355625 T1143 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 355625 T1143 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 355625 T1143 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 355625 T1143 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 355626 T1143 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 355626 T1143 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 355626 T1143 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 355648 T1143 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 355650 T1143 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 355652 T1143 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 355654 T1143 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 355657 T1143 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /schema,/admin/properties,/admin/ping,/update/json,standard,/admin/logging,/admin/segments,/admin/file,/admin/plugins,/admin/luke,/config,/get,/update,/admin/threads,/update/csv,/admin/mbeans,/admin/system,/replication,/update/json/docs
   [junit4]   2> 355659 T1143 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 355661 T1143 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 355661 T1143 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 355663 T1143 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 355663 T1143 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 355663 T1143 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@56bcdcf6
   [junit4]   2> 355664 T1143 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@45fe99f3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4e2d446b; maxCacheMB=0.4970703125 maxMergeSizeMB=0.5986328125)),segFN=segments_1,generation=1}
   [junit4]   2> 355664 T1143 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 355664 T1143 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@330e0bca[collection1] main
   [junit4]   2> 355664 T1143 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 355665 T1143 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 355665 T1143 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 355665 T1143 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 355666 T1143 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 355666 T1143 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 355666 T1143 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 355667 T1143 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 355667 T1143 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 355667 T1143 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 355667 T1144 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@330e0bca[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 355668 T1143 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 355668 T1147 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:44845 collection:control_collection shard:shard1
   [junit4]   2> 355669 T1113 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 355670 T1113 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 355670 T1147 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 355672 T1113 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 355673 T1113 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 355673 T1147 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 355674 T1151 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@45fa6ca0 name:ZooKeeperConnection Watcher:127.0.0.1:44840/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 355674 T1113 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 355675 T1139 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 355675 T1147 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 355675 T1113 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 355676 T1140 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 355676 T1113 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 355675 T1147 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C358 name=collection1 org.apache.solr.core.SolrCore@4df6620 url=http://127.0.0.1:44845/collection1 node=127.0.0.1:44845_ C358_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=http://127.0.0.1:44845, node_name=127.0.0.1:44845_, state=down}
   [junit4]   2> 355676 T1147 C:control_collection S:shard1 C358 P44845 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:44845/collection1/
   [junit4]   2> 355677 T1147 C:control_collection S:shard1 C358 P44845 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 355677 T1147 C:control_collection S:shard1 C358 P44845 oasc.SyncStrategy.syncToMe http://127.0.0.1:44845/collection1/ has no replicas
   [junit4]   2> 355677 T1147 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:44845/collection1/ shard1
   [junit4]   2> 355677 T1139 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 355677 T1151 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 355677 T1147 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 355679 T1113 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 355681 T1139 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 355682 T1140 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:44845",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 355784 T1139 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 355784 T1151 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 355835 T1147 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:44845/collection1/ and leader is http://127.0.0.1:44845/collection1/
   [junit4]   2> 355836 T1147 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:44845
   [junit4]   2> 355836 T1147 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 355836 T1147 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 355836 T1147 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 355837 T1139 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 355838 T1140 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:44845",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:44845_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state"} current state version: 3
   [junit4]   2> 355839 T1140 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:44845",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:44845_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 355941 T1139 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 355941 T1151 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 357336 T1113 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/shard-1-001/cores/collection1
   [junit4]   2> 357339 T1113 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/shard-1-001
   [junit4]   2> 357341 T1113 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 357342 T1113 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:44861
   [junit4]   2> 357343 T1113 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/tempDir-001/jetty1, hostPort=44860, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/shard-1-001/cores}
   [junit4]   2> 357343 T1113 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3972aa3f
   [junit4]   2> 357343 T1113 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/shard-1-001/'
   [junit4]   2> 357389 T1113 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/shard-1-001/solr.xml
   [junit4]   2> 357450 T1113 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/shard-1-001/cores
   [junit4]   2> 357451 T1113 oasc.CoreContainer.<init> New CoreContainer 1582011677
   [junit4]   2> 357451 T1113 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/shard-1-001/]
   [junit4]   2> 357451 T1113 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/shard-1-001/lib
   [junit4]   2> 357452 T1113 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/shard-1-001/lib).
   [junit4]   2> 357462 T1113 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 357462 T1113 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 357462 T1113 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 357463 T1113 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 357463 T1113 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 357463 T1113 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 357463 T1113 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 357464 T1113 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 357464 T1113 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 357464 T1113 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 357464 T1113 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 357465 T1113 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 357465 T1113 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 357465 T1113 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 357465 T1113 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 357465 T1113 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:44840/solr
   [junit4]   2> 357466 T1113 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 357466 T1113 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 357467 T1113 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 357468 T1163 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@10d2866f name:ZooKeeperConnection Watcher:127.0.0.1:44840 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 357469 T1113 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 357469 T1113 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 357471 T1113 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 357472 T1166 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@67824364 name:ZooKeeperConnection Watcher:127.0.0.1:44840/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 357472 T1113 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 357475 T1113 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 358478 T1113 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:44860_
   [junit4]   2> 358479 T1113 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:44860_
   [junit4]   2> 358482 T1113 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 358483 T1113 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/shard-1-001/cores
   [junit4]   2> 358485 T1113 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, name=collection1, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/shard-1-001/cores/collection1/, transient=false, collection=collection1, coreNodeName=, shard=, schema=schema.xml, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/shard-1-001/cores/collection1}
   [junit4]   2> 358486 T1113 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/shard-1-001/cores/collection1/
   [junit4]   2> 358486 T1113 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 358487 T1167 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 358487 T1167 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 358488 T1139 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 358488 T1167 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 358488 T1140 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:44860",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:44860_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 358489 T1140 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:44860",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:44860_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 358489 T1140 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 358489 T1140 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 358490 T1166 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 358490 T1151 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 358490 T1139 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 359488 T1167 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 359489 T1167 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 359489 T1167 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 359490 T1167 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 359490 T1167 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 359491 T1167 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/shard-1-001/cores/collection1/'
   [junit4]   2> 359510 T1167 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 359513 T1167 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 359526 T1167 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 359553 T1167 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 359611 T1167 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 359612 T1167 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 359624 T1167 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 360082 T1167 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 360369 T1167 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 360371 T1167 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 360379 T1167 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 360383 T1167 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 360386 T1167 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 360386 T1167 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 360386 T1167 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 360386 T1167 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 360387 T1167 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 360387 T1167 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 360387 T1167 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 360387 T1167 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 360388 T1167 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 360388 T1167 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6354743
   [junit4]   2> 360388 T1167 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/shard-1-001/cores/collection1/data
   [junit4]   2> 360389 T1167 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 360389 T1167 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 360389 T1167 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 4661B87F5DD5B6B8-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 360389 T1167 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=30, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=96.6904296875, floorSegmentMB=1.7919921875, forceMergeDeletesPctAllowed=1.9750866148602197, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 360390 T1167 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@52f7bece lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@70ff0f0; maxCacheMB=0.4970703125 maxMergeSizeMB=0.5986328125)),segFN=segments_1,generation=1}
   [junit4]   2> 360390 T1167 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 360395 T1167 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 360395 T1167 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 360395 T1167 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 360395 T1167 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 360396 T1167 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 360396 T1167 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 360396 T1167 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 360396 T1167 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 360396 T1167 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 360412 T1167 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 360414 T1167 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 360415 T1167 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 360417 T1167 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 360422 T1167 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /schema,/admin/properties,/admin/ping,/update/json,standard,/admin/logging,/admin/segments,/admin/file,/admin/plugins,/admin/luke,/config,/get,/update,/admin/threads,/update/csv,/admin/mbeans,/admin/system,/replication,/update/json/docs
   [junit4]   2> 360424 T1167 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 360425 T1167 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 360425 T1167 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 360426 T1167 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 360426 T1167 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 360427 T1167 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@2a830fe4
   [junit4]   2> 360427 T1167 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@52f7bece lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@70ff0f0; maxCacheMB=0.4970703125 maxMergeSizeMB=0.5986328125)),segFN=segments_1,generation=1}
   [junit4]   2> 360428 T1167 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 360428 T1167 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@15f2cffa[collection1] main
   [junit4]   2> 360428 T1167 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 360429 T1167 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 360429 T1167 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 360429 T1167 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 360429 T1167 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 360429 T1167 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 360430 T1167 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 360430 T1167 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 360430 T1167 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 360430 T1167 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 360431 T1168 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@15f2cffa[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 360431 T1167 C:collection1 c:collection1 oasc.CoreCon

[...truncated too long message...]

concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 2169151 T41125 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:31843/pva/q
   [junit4]   2> 2169151 T41125 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:31828/pva/q
   [junit4]   2> 2169152 T41125 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:31855/pva/q
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=4661B87F5DD5B6B8 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=th -Dtests.timezone=Etc/GMT+12 -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   69.7s J3 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31843/pva/q/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4661B87F5DD5B6B8:CE3587A5F329DB40]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31843/pva/q/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 54 more
   [junit4]   2> 2169180 T41125 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 4661B87F5DD5B6B8-001
   [junit4]   2> 69707 T41124 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50), sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=th, timezone=Etc/GMT+12
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=132876968,total=359137280
   [junit4]   2> NOTE: All tests run in this JVM: [TestJoin, IndexSchemaRuntimeFieldTest, TestInitQParser, OutOfBoxZkACLAndCredentialsProvidersTest, TestUtils, TestComplexPhraseQParserPlugin, FileUtilsTest, TestPostingsSolrHighlighter, MergeStrategyTest, TestDefaultSearchFieldResource, TestClusterStateMutator, TestArbitraryIndexDir, DateMathParserTest, DistanceUnitsTest, TestSchemaManager, BinaryUpdateRequestHandlerTest, SoftAutoCommitTest, TestLMDirichletSimilarityFactory, TestCSVLoader, TestSearcherReuse, TestQuerySenderListener, TestWordDelimiterFilterFactory, ShardRoutingTest, TestDistributedGrouping, SampleTest, TestJmxIntegration, UnloadDistributedZkTest, TestSolr4Spatial2, ClusterStateTest, TestManagedResource, TestShardHandlerFactory, JsonLoaderTest, TestRandomMergePolicy, SpatialHeatmapFacetsTest, TestValueSourceCache, DefaultValueUpdateProcessorTest, TestConfigReload, TestQueryTypes, DistributedFacetPivotLongTailTest, TestBulkSchemaAPI, CurrencyFieldXmlFileTest, TestRebalanceLeaders, TestManagedSchema, ZkCLITest, ScriptEngineTest, OutputWriterTest, SolrXmlInZkTest, DeleteLastCustomShardedReplicaTest, TestTrackingShardHandlerFactory, InfoHandlerTest, TermVectorComponentDistributedTest, SolrCoreTest, TestGroupingSearch, TestDefaultSimilarityFactory, CollectionsAPIDistributedZkTest, TestUniqueKeyFieldResource, DocValuesMultiTest, CoreMergeIndexesAdminHandlerTest, OverseerTest, LeaderElectionIntegrationTest, RecoveryZkTest, FullSolrCloudDistribCmdsTest, ClusterStateUpdateTest, LeaderElectionTest, ZkSolrClientTest, TestZkChroot, TestDistributedSearch, TestHighlightDedupGrouping, DistribCursorPagingTest, DistribJoinFromCollectionTest, MultiThreadedOCPTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J3 in 69.98s, 1 test, 1 error <<< FAILURES!

[...truncated 411 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 4 errors, 1 failure, 57 ignored (8 assumptions)

Total time: 167 minutes 6 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 837990325 bytes
Compression is 0.0%
Took 2 min 44 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 804 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/804/

4 tests failed.
FAILED:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=867, name=updateExecutor-67-thread-4, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=867, name=updateExecutor-67-thread-4, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]
Caused by: java.util.concurrent.RejectedExecutionException: Task org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner@7da8d7ad rejected from java.util.concurrent.ThreadPoolExecutor@57e7df4e[Shutting down, pool size = 1, active threads = 1, queued tasks = 0, completed tasks = 1872]
	at __randomizedtesting.SeedInfo.seed([8B79B66012CE5D28]:0)
	at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2048)
	at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:821)
	at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1372)
	at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient$Runner.run(ConcurrentUpdateSolrClient.java:278)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:32408/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:32408/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([8B79B66012CE5D28:32D89BABC3230D0]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:132)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:64)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([8B79B66012CE5D28:32D89BABC3230D0]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([8B79B66012CE5D28:32D89BABC3230D0]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9892 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/init-core-data-001
   [junit4]   2> 10792 T41 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 10826 T41 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 10827 T42 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 10928 T41 oasc.ZkTestServer.run start zk server on port:44554
   [junit4]   2> 10930 T41 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 10931 T41 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 10936 T49 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6c490226 name:ZooKeeperConnection Watcher:127.0.0.1:44554 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 10936 T41 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 10938 T41 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 10938 T41 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 10942 T41 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 10944 T41 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 10946 T52 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@58b5319d name:ZooKeeperConnection Watcher:127.0.0.1:44554/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 10947 T41 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 10948 T41 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 10948 T41 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 10954 T41 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 10958 T41 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 10962 T41 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 10967 T41 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 11035 T41 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 11041 T41 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 11042 T41 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 11047 T41 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 11048 T41 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 11052 T41 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 11053 T41 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 11057 T41 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 11058 T41 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 11062 T41 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 11063 T41 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 11068 T41 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 11069 T41 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 11073 T41 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 11074 T41 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 11079 T41 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 11081 T41 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 11086 T41 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 11087 T41 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 11091 T41 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 11092 T41 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 13397 T41 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/control-001/cores/collection1
   [junit4]   2> 14572 T41 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 15219 T41 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:42175
   [junit4]   2> 15220 T41 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/tempDir-001/control/data, hostPort=14507}
   [junit4]   2> 15936 T41 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 16130 T41 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/control-001/'
   [junit4]   2> 17301 T41 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/control-001/solr.xml
   [junit4]   2> 17583 T41 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/control-001/cores
   [junit4]   2> 18077 T41 oasc.CoreContainer.<init> New CoreContainer 89356419
   [junit4]   2> 18078 T41 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/control-001/]
   [junit4]   2> 18079 T41 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/control-001/lib
   [junit4]   2> 18081 T41 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/control-001/lib).
   [junit4]   2> 18440 T41 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 18441 T41 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 18459 T41 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 18460 T41 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 18461 T41 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 18462 T41 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 18463 T41 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 18463 T41 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 18464 T41 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 18465 T41 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 18466 T41 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 19343 T41 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 19370 T41 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 19381 T41 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 19396 T41 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 19397 T41 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:44554/solr
   [junit4]   2> 19398 T41 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 19399 T41 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 19401 T41 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 19407 T65 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@278c4835 name:ZooKeeperConnection Watcher:127.0.0.1:44554 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 19408 T41 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 19410 T41 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 19444 T41 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 19473 T68 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5c7a92e6 name:ZooKeeperConnection Watcher:127.0.0.1:44554/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 19474 T41 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 19501 T41 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 19521 T41 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 19539 T41 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 19545 T41 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 19552 T41 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 19568 T41 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 19572 T41 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 19575 T41 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 19578 T41 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:14507_
   [junit4]   2> 19591 T41 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:14507_
   [junit4]   2> 19595 T41 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 19598 T41 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 19601 T41 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 19612 T41 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:14507_
   [junit4]   2> 19613 T41 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 19616 T41 oasc.Overseer.start Overseer (id=93578916856594435-127.0.0.1:14507_-n_0000000000) starting
   [junit4]   2> 19647 T41 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 19749 T41 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 20475 T70 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 20475 T41 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 20486 T69 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 20920 T41 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/control-001/cores
   [junit4]   2> 21411 T41 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, schema=schema.xml, collection=control_collection, name=collection1, dataDir=data/, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/control-001/cores/collection1, loadOnStartup=true, shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/control-001/cores/collection1/, coreNodeName=}
   [junit4]   2> 21422 T41 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/control-001/cores/collection1/
   [junit4]   2> 21422 T41 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 21441 T72 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 21441 T72 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 21452 T68 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 21452 T72 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 21461 T69 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:14507_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14507",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 21524 T69 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:14507_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14507",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 21534 T69 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 21567 T69 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 21570 T68 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 22454 T72 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 22454 T72 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 22455 T72 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 22456 T72 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 22457 T72 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 22465 T72 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/control-001/cores/collection1/'
   [junit4]   2> 22530 T72 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 22563 T72 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 22642 T72 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 22784 T72 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 23097 T72 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 23149 T72 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 23312 T72 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 26141 T72 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 26307 T72 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 26321 T72 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 27817 T72 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 27846 T72 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 27853 T72 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 27861 T72 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 27862 T72 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 27862 T72 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 27864 T72 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 27865 T72 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 27884 T72 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 27885 T72 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 27914 T72 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/control-001/cores/collection1/data/
   [junit4]   2> 27926 T72 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@38bf66e5
   [junit4]   2> 28270 T72 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/control-001/cores/collection1/data
   [junit4]   2> 28271 T72 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/control-001/cores/collection1/data/index/
   [junit4]   2> 28271 T72 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 28272 T72 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/control-001/cores/collection1/data/index
   [junit4]   2> 28275 T72 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=7, maxMergeAtOnceExplicit=2, maxMergedSegmentMB=70.2548828125, floorSegmentMB=1.7314453125, forceMergeDeletesPctAllowed=8.17227071920021, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 28567 T72 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3bbfd45b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2f0ec102),segFN=segments_1,generation=1}
   [junit4]   2> 28573 T72 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 30684 T72 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 30968 T72 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 30999 T72 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 31000 T72 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 31001 T72 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 31002 T72 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 31210 T72 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 31211 T72 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 31212 T72 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 31675 T72 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 31738 T72 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 31743 T72 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 31748 T72 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 32175 T72 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/json/docs,/update,/admin/file,/admin/properties,/get,standard,/admin/segments,/schema,/admin/system,/admin/logging,/update/csv,/config,/admin/mbeans,/admin/plugins,/admin/luke,/replication,/admin/ping,/update/json,/admin/threads
   [junit4]   2> 32185 T72 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 32320 T72 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 32321 T72 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 32375 T72 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 32376 T72 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 32382 T72 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@7e138f1b
   [junit4]   2> 32409 T72 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@3bbfd45b lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2f0ec102),segFN=segments_1,generation=1}
   [junit4]   2> 32410 T72 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 32582 T72 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1a1d85b8[collection1] main
   [junit4]   2> 32638 T72 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 32641 T72 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 32642 T72 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 32660 T72 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 32661 T72 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 32794 T72 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 32796 T72 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 32797 T72 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 32798 T72 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 33023 T72 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 33032 T73 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1a1d85b8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 33041 T72 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 33048 T76 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:14507 collection:control_collection shard:shard1
   [junit4]   2> 33049 T41 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 33050 T41 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 33094 T76 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 33104 T76 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 33107 T68 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 33108 T76 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 33109 T76 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 33109 T69 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 33124 T68 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> ASYNC  NEW_CORE C0 name=collection1 org.apache.solr.core.SolrCore@1ffc159e url=http://127.0.0.1:14507/collection1 node=127.0.0.1:14507_ C0_STATE=coll:control_collection core:collection1 props:{state=down, node_name=127.0.0.1:14507_, core=collection1, base_url=http://127.0.0.1:14507}
   [junit4]   2> 33135 T76 C:control_collection S:shard1 C0 P14507 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:14507/collection1/
   [junit4]   2> 33143 T76 C:control_collection S:shard1 C0 P14507 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 33144 T76 C:control_collection S:shard1 C0 P14507 oasc.SyncStrategy.syncToMe http://127.0.0.1:14507/collection1/ has no replicas
   [junit4]   2> 33145 T76 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:14507/collection1/ shard1
   [junit4]   2> 33162 T76 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 33172 T68 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 33175 T69 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14507",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 33185 T41 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 33187 T41 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 33190 T80 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@aac04ed name:ZooKeeperConnection Watcher:127.0.0.1:44554/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 33190 T41 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 33192 T41 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 33193 T41 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 33205 T41 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 33285 T68 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 33285 T80 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 33327 T76 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:14507/collection1/ and leader is http://127.0.0.1:14507/collection1/
   [junit4]   2> 33340 T76 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:14507
   [junit4]   2> 33341 T76 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 33341 T76 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 33342 T76 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 33345 T68 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 33348 T69 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:14507_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14507",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state"} current state version: 3
   [junit4]   2> 33350 T69 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:14507_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:14507",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 33459 T80 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 33459 T68 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 35292 T41 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/shard-1-001/cores/collection1
   [junit4]   2> 35297 T41 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/shard-1-001
   [junit4]   2> 35298 T41 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 35302 T41 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:35793
   [junit4]   2> 35303 T41 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=32030, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/tempDir-001/jetty1, hostContext=/, solrconfig=solrconfig.xml}
   [junit4]   2> 35304 T41 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 35305 T41 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/shard-1-001/'
   [junit4]   2> 35388 T41 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/shard-1-001/solr.xml
   [junit4]   2> 35502 T41 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/shard-1-001/cores
   [junit4]   2> 35503 T41 oasc.CoreContainer.<init> New CoreContainer 1405982278
   [junit4]   2> 35504 T41 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/shard-1-001/]
   [junit4]   2> 35504 T41 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/shard-1-001/lib
   [junit4]   2> 35506 T41 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/shard-1-001/lib).
   [junit4]   2> 35529 T41 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 35530 T41 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 35530 T41 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 35531 T41 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 35532 T41 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 35533 T41 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 35534 T41 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 35534 T41 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 35535 T41 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 35536 T41 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 35537 T41 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 35538 T41 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 35539 T41 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 35540 T41 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 35541 T41 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 35541 T41 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:44554/solr
   [junit4]   2> 35542 T41 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 35543 T41 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 35544 T41 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 35548 T92 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7fa6e8b1 name:ZooKeeperConnection Watcher:127.0.0.1:44554 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 35549 T41 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 35551 T41 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 35555 T41 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 35559 T95 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5f50ba51 name:ZooKeeperConnection Watcher:127.0.0.1:44554/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 35559 T41 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 35573 T41 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 36585 T41 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32030_
   [junit4]   2> 36588 T41 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32030_
   [junit4]   2> 36596 T41 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 36610 T41 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/shard-1-001/cores
   [junit4]   2> 36614 T41 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, name=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/shard-1-001/cores/collection1, dataDir=data/, config=solrconfig.xml, shard=, collection=collection1, schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/shard-1-001/cores/collection1/, transient=false, coreNodeName=}
   [junit4]   2> 36615 T41 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/shard-1-001/cores/collection1/
   [junit4]   2> 36616 T41 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 36618 T96 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 36618 T96 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 36621 T96 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 36621 T68 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 36624 T69 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:32030_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32030",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 36626 T69 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:32030_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32030",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 36626 T69 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 36627 T69 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 36631 T68 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 36631 T80 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 36631 T95 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 37621 T96 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 37622 T96 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 37624 T96 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 37625 T96 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 37627 T96 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 37627 T96 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/shard-1-001/cores/collection1/'
   [junit4]   2> 37656 T96 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 37661 T96 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 37697 T96 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 37745 T96 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.2.0
   [junit4]   2> 37861 T96 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 37864 T96 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 38009 T96 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 38677 T96 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 38702 T96 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 38705 T96 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 38738 T96 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 38744 T96 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 38750 T96 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 38751 T96 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 38752 T96 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 38752 T96 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 38754 T96 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 38754 T96 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 38755 T96 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 38755 T96 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 38756 T96 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 38756 T96 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@38bf66e5
   [junit4]   2> 38758 T96 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/shard-1-001/cores/collection1/data
   [junit4]   2> 38758 T96 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 38759 T96 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 38759 T96 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 8B79B66012CE5D28-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 38760 T96 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=7, maxMergeAtOnceExplicit=2, maxMergedSegmentMB=70.2548828125, floorSegmentMB=1.7314453125, forceMergeDeletesPctAllowed=8.17227071920021, segmentsPerTier=14.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 38761 T96 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7bb9d2a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2e4b345d),segFN=segments_1,generation=1}
   [junit4]   2> 38761 T96 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 38774 T96 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 38775 T96 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 38775 T96 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 38776 T96 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 38776 T96 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 38777 T96 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 38777 T96 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 38778 T96 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 38778 T96 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 38800 T96 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 38804 T96 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 38807 T96 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 38812 T96 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 38852 T96 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/json/docs,/update,/admin/file,/admin/properties,/get,standard,/admin/segments,/schema,/admin/system,/admin/logging,/update/csv,/config,/admin/mbeans,/admin/plugins,/admin/luke,/replication,/admin/ping,/update/json,/admin/threads
   [junit4]   2> 38855 T96 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 38856 T96 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 38857 T96 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 38859 T96 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 38859 T96 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 38861 T96 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@42e13ec6
   [junit4]   2> 38862 T96 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@7bb9d2a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@2e4b345d),segFN=segments_1,generation=1}
   [junit4]   2> 38862 T96 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 38863 T96 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@64903dcb[collection1] main
   [junit4]   2> 38863 T96 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 38865 T96 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 38866 T96 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 38866 T96 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 38867 T96 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 38867 T96 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 38868 T96 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 38869 T96 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 38869 T96 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 38870 T96 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 38874 T97 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@64903dcb[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 38876 T96 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 38877 T100 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:32030 collection:collection1 shard:shard1
   [junit4]   2> 38878 T41 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 38879 T100 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 38879 T41 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 38889 T100 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 38892 T68 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 38893 T100 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 38893 T100 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C1 name=collection1 org.apache.solr.core.SolrCore@5136a826 url=http://127.0.0.1:32030/collection1 node=127.0.0.1:32030_ C1_STATE=coll:collection1 core:collection1 props:{state=down, node_name=127.0.0.1:32030_, core=collection1, base_url=http://127.0.0.1:32030}
   [junit4]   2> 38893 T100 C:collection1 S:shard1 C1 P32030 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:32030/collection1/
   [junit4]   2> 38894 T100 C:collection1 S:shard1 C1 P32030 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 38894 T69 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 38894 T100 C:collection1 S:shard1 C1 P32030 oasc.SyncStrategy.syncToMe http://127.0.0.1:32030/collection1/ has no replicas
   [junit4]   2> 38895 T100 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:32030/collection1/ shard1
   [junit4]   2> 38896 T100 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 38898 T80 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 38898 T95 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 38898 T68 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 38907 T69 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32030",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 38911 T68 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 38914 T80 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 38915 T68 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 38915 T95 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 38956 T100 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:32030/collection1/ and leader is http://127.0.0.1:32030/collection1/
   [junit4]   2> 38957 T100 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:32030
   [junit4]   2> 38958 T100 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 38958 T100 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 38959 T100 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 38961 T68 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 38964 T69 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32030_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32030",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state"} current state version: 7
   [junit4]   2> 38965 T69 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:32030_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32030",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 39072 T80 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 39072 T68 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 39073 T95 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 40595 T41 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecov

[...truncated too long message...]

ns to: http://127.0.0.1:12152/
   [junit4]   2> 2311164 T5589 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:12168/
   [junit4]   2> 2311164 T5589 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:12182/
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=8B79B66012CE5D28 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=zh_HK -Dtests.timezone=US/Hawaii -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   76.3s J2 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([8B79B66012CE5D28:32D89BABC3230D0]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 2311179 T5589 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest 8B79B66012CE5D28-001
   [junit4]   2> 76266 T5588 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene50, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=zh_HK, timezone=US/Hawaii
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=150590656,total=432537600
   [junit4]   2> NOTE: All tests run in this JVM: [TestSchemaNameResource, TestDocBasedVersionConstraints, TestReqParamsAPI, CachingDirectoryFactoryTest, TestFieldSortValues, CursorPagingTest, TestBM25SimilarityFactory, TestLazyCores, TestHdfsUpdateLog, TestCSVResponseWriter, TestExceedMaxTermLength, CursorMarkTest, RemoteQueryErrorTest, BasicDistributedZk2Test, TestNamedUpdateProcessors, HdfsUnloadDistributedZkTest, FileBasedSpellCheckerTest, SolrCoreCheckLockOnStartupTest, AliasIntegrationTest, URLClassifyProcessorTest, TestSolrQueryParser, CurrencyFieldOpenExchangeTest, SolrTestCaseJ4Test, TestLRUCache, DocumentBuilderTest, TestComponentsName, TestOverriddenPrefixQueryForCustomFieldType, RAMDirectoryFactoryTest, TestRealTimeGet, DistributedDebugComponentTest, TestFastWriter, TestChildDocTransformer, DateFieldTest, BadComponentTest, TestManagedSchemaDynamicFieldResource, TestRTGBase, ResponseHeaderTest, TestCryptoKeys, ExitableDirectoryReaderTest, TestReversedWildcardFilterFactory, SimpleMLTQParserTest, TestRemoteStreaming, TestLuceneMatchVersion, TestMergePolicyConfig, DistribCursorPagingTest, TestSuggestSpellingConverter, TestAddFieldRealTimeGet, TestSolrXml, SolrPluginUtilsTest, TestCloudPivotFacet, SaslZkACLProviderTest, TestExpandComponent, BadCopyFieldTest, StatsComponentTest, SliceStateTest, TestHashPartitioner, DistributedSuggestComponentTest, OpenCloseCoreStressTest, TestFoldingMultitermQuery, TestTrie, NoCacheHeaderTest, WordBreakSolrSpellCheckerTest, TestPseudoReturnFields, TestWordDelimiterFilterFactory, DirectUpdateHandlerOptimizeTest, StatelessScriptUpdateProcessorFactoryTest, SortByFunctionTest, DistanceFunctionTest, DebugComponentTest, IndexBasedSpellCheckerTest, StandardRequestHandlerTest, TestWriterPerf, TestOmitPositions, TermVectorComponentTest, RequiredFieldsTest, LoggingHandlerTest, IndexSchemaTest, ReturnFieldsTest, JsonLoaderTest, SearchHandlerTest, HighlighterConfigTest, TestQuerySenderNoQuery, SolrIndexConfigTest, SampleTest, TestDocSet, TestBinaryField, TestSearchPerf, MinimalSchemaTest, TestElisionMultitermQuery, TestConfig, OutputWriterTest, TestFuzzyAnalyzedSuggestions, TestSolrCoreProperties, TestPostingsSolrHighlighter, DirectSolrConnectionTest, NotRequiredUniqueKeyTest, TestPhraseSuggestions, SpellPossibilityIteratorTest, TestCharFilters, SynonymTokenizerTest, EchoParamsTest, TestDFRSimilarityFactory, TestPerFieldSimilarity, TestLMJelinekMercerSimilarityFactory, TestDefaultSimilarityFactory, TestJmxMonitoredMap, OpenExchangeRatesOrgProviderTest, DateMathParserTest, ChaosMonkeyNothingIsSafeTest, MultiThreadedOCPTest, ReplicationFactorTest]
   [junit4] Completed on J2 in 77.05s, 1 test, 1 error <<< FAILURES!

[...truncated 315 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 4 errors, 57 ignored (8 assumptions)

Total time: 144 minutes 6 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 880036426 bytes
Compression is 0.0%
Took 2 min 38 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 803 - Still Failing

Posted by Robert Muir <rc...@gmail.com>.
I'm looking into this one next.

On Tue, Mar 31, 2015 at 1:06 PM, Apache Jenkins Server
<je...@builds.apache.org> wrote:
> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/803/
>
> 1 tests failed.
> FAILED:  junit.framework.TestSuite.org.apache.lucene.replicator.IndexAndTaxonomyReplicationClientTest
>
> Error Message:
> file handle leaks: [FileChannel(/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/replicator/test/J2/temp/lucene.replicator.IndexAndTaxonomyReplicationClientTest 10CCB411FDA8B966-001/index-MMapDirectory-001/_1j_MockRandom_0.tio)]
>
> Stack Trace:
> java.lang.RuntimeException: file handle leaks: [FileChannel(/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/replicator/test/J2/temp/lucene.replicator.IndexAndTaxonomyReplicationClientTest 10CCB411FDA8B966-001/index-MMapDirectory-001/_1j_MockRandom_0.tio)]
>         at __randomizedtesting.SeedInfo.seed([10CCB411FDA8B966]:0)
>         at org.apache.lucene.mockfile.LeakFS.onClose(LeakFS.java:64)
>         at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:78)
>         at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:79)
>         at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:79)
>         at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:79)
>         at org.apache.lucene.util.TestRuleTemporaryFilesCleanup.afterAlways(TestRuleTemporaryFilesCleanup.java:212)
>         at com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterAlways(TestRuleAdapter.java:31)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
> Caused by: java.lang.Exception
>         at org.apache.lucene.mockfile.LeakFS.onOpen(LeakFS.java:47)
>         at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>         at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:191)
>         at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:204)
>         at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:163)
>         at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:204)
>         at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:163)
>         at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:204)
>         at java.nio.channels.FileChannel.open(FileChannel.java:287)
>         at java.nio.channels.FileChannel.open(FileChannel.java:334)
>         at org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:213)
>         at org.apache.lucene.codecs.blocktreeords.OrdsBlockTreeTermsReader.<init>(OrdsBlockTreeTermsReader.java:71)
>         at org.apache.lucene.codecs.mockrandom.MockRandomPostingsFormat.fieldsProducer(MockRandomPostingsFormat.java:383)
>         at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:258)
>         at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:336)
>         at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:104)
>         at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:65)
>         at org.apache.lucene.index.StandardDirectoryReader$1.doBody(StandardDirectoryReader.java:58)
>         at org.apache.lucene.index.StandardDirectoryReader$1.doBody(StandardDirectoryReader.java:50)
>         at org.apache.lucene.index.SegmentInfos$FindSegmentsFile.run(SegmentInfos.java:660)
>         at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:50)
>         at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:63)
>         at org.apache.lucene.replicator.IndexAndTaxonomyReplicationClientTest.assertHandlerRevision(IndexAndTaxonomyReplicationClientTest.java:158)
>         at org.apache.lucene.replicator.IndexAndTaxonomyReplicationClientTest.testConsistencyOnExceptions(IndexAndTaxonomyReplicationClientTest.java:477)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         ... 8 more
>
>
>
>
> Build Log:
> [...truncated 8356 lines...]
>    [junit4] Suite: org.apache.lucene.replicator.IndexAndTaxonomyReplicationClientTest
>    [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {$full_path$=PostingsFormat(name=Asserting), $payloads$=BlockTreeOrds(blocksize=128), $facets=PostingsFormat(name=MockRandom)}, docValues:{$facets=DocValuesFormat(name=Lucene50)}, sim=DefaultSimilarity, locale=hu_HU, timezone=America/Scoresbysund
>    [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=197578352,total=245366784
>    [junit4]   2> NOTE: All tests run in this JVM: [SessionTokenTest, IndexAndTaxonomyReplicationClientTest]
>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=IndexAndTaxonomyReplicationClientTest -Dtests.seed=10CCB411FDA8B966 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=hu_HU -Dtests.timezone=America/Scoresbysund -Dtests.asserts=true -Dtests.file.encoding=UTF-8
>    [junit4] ERROR   0.00s J2 | IndexAndTaxonomyReplicationClientTest (suite) <<<
>    [junit4]    > Throwable #1: java.lang.RuntimeException: file handle leaks: [FileChannel(/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/replicator/test/J2/temp/lucene.replicator.IndexAndTaxonomyReplicationClientTest 10CCB411FDA8B966-001/index-MMapDirectory-001/_1j_MockRandom_0.tio)]
>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([10CCB411FDA8B966]:0)
>    [junit4]    >        at org.apache.lucene.mockfile.LeakFS.onClose(LeakFS.java:64)
>    [junit4]    >        at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:78)
>    [junit4]    >        at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:79)
>    [junit4]    >        at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:79)
>    [junit4]    >        at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:79)
>    [junit4]    >        at java.lang.Thread.run(Thread.java:745)
>    [junit4]    > Caused by: java.lang.Exception
>    [junit4]    >        at org.apache.lucene.mockfile.LeakFS.onOpen(LeakFS.java:47)
>    [junit4]    >        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]    >        at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:191)
>    [junit4]    >        at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:204)
>    [junit4]    >        at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:163)
>    [junit4]    >        at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:204)
>    [junit4]    >        at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:163)
>    [junit4]    >        at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:204)
>    [junit4]    >        at java.nio.channels.FileChannel.open(FileChannel.java:287)
>    [junit4]    >        at java.nio.channels.FileChannel.open(FileChannel.java:334)
>    [junit4]    >        at org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:213)
>    [junit4]    >        at org.apache.lucene.codecs.blocktreeords.OrdsBlockTreeTermsReader.<init>(OrdsBlockTreeTermsReader.java:71)
>    [junit4]    >        at org.apache.lucene.codecs.mockrandom.MockRandomPostingsFormat.fieldsProducer(MockRandomPostingsFormat.java:383)
>    [junit4]    >        at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:258)
>    [junit4]    >        at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:336)
>    [junit4]    >        at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:104)
>    [junit4]    >        at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:65)
>    [junit4]    >        at org.apache.lucene.index.StandardDirectoryReader$1.doBody(StandardDirectoryReader.java:58)
>    [junit4]    >        at org.apache.lucene.index.StandardDirectoryReader$1.doBody(StandardDirectoryReader.java:50)
>    [junit4]    >        at org.apache.lucene.index.SegmentInfos$FindSegmentsFile.run(SegmentInfos.java:660)
>    [junit4]    >        at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:50)
>    [junit4]    >        at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:63)
>    [junit4]    >        at org.apache.lucene.replicator.IndexAndTaxonomyReplicationClientTest.assertHandlerRevision(IndexAndTaxonomyReplicationClientTest.java:158)
>    [junit4]    >        at org.apache.lucene.replicator.IndexAndTaxonomyReplicationClientTest.testConsistencyOnExceptions(IndexAndTaxonomyReplicationClientTest.java:477)
>    [junit4]    >        ... 8 more
>    [junit4] Completed on J2 in 42.96s, 5 tests, 1 error <<< FAILURES!
>
> [...truncated 12 lines...]
> BUILD FAILED
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:456: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:2166: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/module-build.xml:58: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 7 suites, 31 tests, 1 suite-level error
>
> Total time: 104 minutes 58 seconds
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
> Archived 5 artifacts
> Archive block size is 32768
> Received 0 blocks and 109219 bytes
> Compression is 0.0%
> Took 52 ms
> Recording test results
> Email was triggered for: Failure
> Sending email for trigger: Failure
>
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


Re: [JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 803 - Still Failing

Posted by Robert Muir <rc...@gmail.com>.
I picked same JVM version and args and revision number, and tried
beasting the seed and master seed. I cannot make this one fail. I
don't see a leak in OrdsBlockTreeTermsReader at a glance.

The test itself is provoking random low level exceptions and
assertHandlerRevision is masking some of them, which worries me a
little that there is a race somewhere and its a real problem.
I will improve the exception on leaked files to not just show the
first one but give more debugging information in case it happens
again.

On Tue, Mar 31, 2015 at 1:06 PM, Apache Jenkins Server
<je...@builds.apache.org> wrote:
> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/803/
>
> 1 tests failed.
> FAILED:  junit.framework.TestSuite.org.apache.lucene.replicator.IndexAndTaxonomyReplicationClientTest
>
> Error Message:
> file handle leaks: [FileChannel(/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/replicator/test/J2/temp/lucene.replicator.IndexAndTaxonomyReplicationClientTest 10CCB411FDA8B966-001/index-MMapDirectory-001/_1j_MockRandom_0.tio)]
>
> Stack Trace:
> java.lang.RuntimeException: file handle leaks: [FileChannel(/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/replicator/test/J2/temp/lucene.replicator.IndexAndTaxonomyReplicationClientTest 10CCB411FDA8B966-001/index-MMapDirectory-001/_1j_MockRandom_0.tio)]
>         at __randomizedtesting.SeedInfo.seed([10CCB411FDA8B966]:0)
>         at org.apache.lucene.mockfile.LeakFS.onClose(LeakFS.java:64)
>         at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:78)
>         at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:79)
>         at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:79)
>         at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:79)
>         at org.apache.lucene.util.TestRuleTemporaryFilesCleanup.afterAlways(TestRuleTemporaryFilesCleanup.java:212)
>         at com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterAlways(TestRuleAdapter.java:31)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
> Caused by: java.lang.Exception
>         at org.apache.lucene.mockfile.LeakFS.onOpen(LeakFS.java:47)
>         at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>         at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:191)
>         at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:204)
>         at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:163)
>         at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:204)
>         at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:163)
>         at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:204)
>         at java.nio.channels.FileChannel.open(FileChannel.java:287)
>         at java.nio.channels.FileChannel.open(FileChannel.java:334)
>         at org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:213)
>         at org.apache.lucene.codecs.blocktreeords.OrdsBlockTreeTermsReader.<init>(OrdsBlockTreeTermsReader.java:71)
>         at org.apache.lucene.codecs.mockrandom.MockRandomPostingsFormat.fieldsProducer(MockRandomPostingsFormat.java:383)
>         at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:258)
>         at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:336)
>         at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:104)
>         at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:65)
>         at org.apache.lucene.index.StandardDirectoryReader$1.doBody(StandardDirectoryReader.java:58)
>         at org.apache.lucene.index.StandardDirectoryReader$1.doBody(StandardDirectoryReader.java:50)
>         at org.apache.lucene.index.SegmentInfos$FindSegmentsFile.run(SegmentInfos.java:660)
>         at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:50)
>         at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:63)
>         at org.apache.lucene.replicator.IndexAndTaxonomyReplicationClientTest.assertHandlerRevision(IndexAndTaxonomyReplicationClientTest.java:158)
>         at org.apache.lucene.replicator.IndexAndTaxonomyReplicationClientTest.testConsistencyOnExceptions(IndexAndTaxonomyReplicationClientTest.java:477)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         ... 8 more
>
>
>
>
> Build Log:
> [...truncated 8356 lines...]
>    [junit4] Suite: org.apache.lucene.replicator.IndexAndTaxonomyReplicationClientTest
>    [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {$full_path$=PostingsFormat(name=Asserting), $payloads$=BlockTreeOrds(blocksize=128), $facets=PostingsFormat(name=MockRandom)}, docValues:{$facets=DocValuesFormat(name=Lucene50)}, sim=DefaultSimilarity, locale=hu_HU, timezone=America/Scoresbysund
>    [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=197578352,total=245366784
>    [junit4]   2> NOTE: All tests run in this JVM: [SessionTokenTest, IndexAndTaxonomyReplicationClientTest]
>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=IndexAndTaxonomyReplicationClientTest -Dtests.seed=10CCB411FDA8B966 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=hu_HU -Dtests.timezone=America/Scoresbysund -Dtests.asserts=true -Dtests.file.encoding=UTF-8
>    [junit4] ERROR   0.00s J2 | IndexAndTaxonomyReplicationClientTest (suite) <<<
>    [junit4]    > Throwable #1: java.lang.RuntimeException: file handle leaks: [FileChannel(/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/replicator/test/J2/temp/lucene.replicator.IndexAndTaxonomyReplicationClientTest 10CCB411FDA8B966-001/index-MMapDirectory-001/_1j_MockRandom_0.tio)]
>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([10CCB411FDA8B966]:0)
>    [junit4]    >        at org.apache.lucene.mockfile.LeakFS.onClose(LeakFS.java:64)
>    [junit4]    >        at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:78)
>    [junit4]    >        at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:79)
>    [junit4]    >        at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:79)
>    [junit4]    >        at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:79)
>    [junit4]    >        at java.lang.Thread.run(Thread.java:745)
>    [junit4]    > Caused by: java.lang.Exception
>    [junit4]    >        at org.apache.lucene.mockfile.LeakFS.onOpen(LeakFS.java:47)
>    [junit4]    >        at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
>    [junit4]    >        at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:191)
>    [junit4]    >        at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:204)
>    [junit4]    >        at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:163)
>    [junit4]    >        at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:204)
>    [junit4]    >        at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:163)
>    [junit4]    >        at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:204)
>    [junit4]    >        at java.nio.channels.FileChannel.open(FileChannel.java:287)
>    [junit4]    >        at java.nio.channels.FileChannel.open(FileChannel.java:334)
>    [junit4]    >        at org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:213)
>    [junit4]    >        at org.apache.lucene.codecs.blocktreeords.OrdsBlockTreeTermsReader.<init>(OrdsBlockTreeTermsReader.java:71)
>    [junit4]    >        at org.apache.lucene.codecs.mockrandom.MockRandomPostingsFormat.fieldsProducer(MockRandomPostingsFormat.java:383)
>    [junit4]    >        at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:258)
>    [junit4]    >        at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:336)
>    [junit4]    >        at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:104)
>    [junit4]    >        at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:65)
>    [junit4]    >        at org.apache.lucene.index.StandardDirectoryReader$1.doBody(StandardDirectoryReader.java:58)
>    [junit4]    >        at org.apache.lucene.index.StandardDirectoryReader$1.doBody(StandardDirectoryReader.java:50)
>    [junit4]    >        at org.apache.lucene.index.SegmentInfos$FindSegmentsFile.run(SegmentInfos.java:660)
>    [junit4]    >        at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:50)
>    [junit4]    >        at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:63)
>    [junit4]    >        at org.apache.lucene.replicator.IndexAndTaxonomyReplicationClientTest.assertHandlerRevision(IndexAndTaxonomyReplicationClientTest.java:158)
>    [junit4]    >        at org.apache.lucene.replicator.IndexAndTaxonomyReplicationClientTest.testConsistencyOnExceptions(IndexAndTaxonomyReplicationClientTest.java:477)
>    [junit4]    >        ... 8 more
>    [junit4] Completed on J2 in 42.96s, 5 tests, 1 error <<< FAILURES!
>
> [...truncated 12 lines...]
> BUILD FAILED
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:456: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:2166: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/module-build.xml:58: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 7 suites, 31 tests, 1 suite-level error
>
> Total time: 104 minutes 58 seconds
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
> Archived 5 artifacts
> Archive block size is 32768
> Received 0 blocks and 109219 bytes
> Compression is 0.0%
> Took 52 ms
> Recording test results
> Email was triggered for: Failure
> Sending email for trigger: Failure
>
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 803 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/803/

1 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.lucene.replicator.IndexAndTaxonomyReplicationClientTest

Error Message:
file handle leaks: [FileChannel(/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/replicator/test/J2/temp/lucene.replicator.IndexAndTaxonomyReplicationClientTest 10CCB411FDA8B966-001/index-MMapDirectory-001/_1j_MockRandom_0.tio)]

Stack Trace:
java.lang.RuntimeException: file handle leaks: [FileChannel(/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/replicator/test/J2/temp/lucene.replicator.IndexAndTaxonomyReplicationClientTest 10CCB411FDA8B966-001/index-MMapDirectory-001/_1j_MockRandom_0.tio)]
	at __randomizedtesting.SeedInfo.seed([10CCB411FDA8B966]:0)
	at org.apache.lucene.mockfile.LeakFS.onClose(LeakFS.java:64)
	at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:78)
	at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:79)
	at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:79)
	at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:79)
	at org.apache.lucene.util.TestRuleTemporaryFilesCleanup.afterAlways(TestRuleTemporaryFilesCleanup.java:212)
	at com.carrotsearch.randomizedtesting.rules.TestRuleAdapter$1.afterAlways(TestRuleAdapter.java:31)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:43)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.Exception
	at org.apache.lucene.mockfile.LeakFS.onOpen(LeakFS.java:47)
	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
	at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:191)
	at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:204)
	at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:163)
	at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:204)
	at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:163)
	at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:204)
	at java.nio.channels.FileChannel.open(FileChannel.java:287)
	at java.nio.channels.FileChannel.open(FileChannel.java:334)
	at org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:213)
	at org.apache.lucene.codecs.blocktreeords.OrdsBlockTreeTermsReader.<init>(OrdsBlockTreeTermsReader.java:71)
	at org.apache.lucene.codecs.mockrandom.MockRandomPostingsFormat.fieldsProducer(MockRandomPostingsFormat.java:383)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:258)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:336)
	at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:104)
	at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:65)
	at org.apache.lucene.index.StandardDirectoryReader$1.doBody(StandardDirectoryReader.java:58)
	at org.apache.lucene.index.StandardDirectoryReader$1.doBody(StandardDirectoryReader.java:50)
	at org.apache.lucene.index.SegmentInfos$FindSegmentsFile.run(SegmentInfos.java:660)
	at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:50)
	at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:63)
	at org.apache.lucene.replicator.IndexAndTaxonomyReplicationClientTest.assertHandlerRevision(IndexAndTaxonomyReplicationClientTest.java:158)
	at org.apache.lucene.replicator.IndexAndTaxonomyReplicationClientTest.testConsistencyOnExceptions(IndexAndTaxonomyReplicationClientTest.java:477)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	... 8 more




Build Log:
[...truncated 8356 lines...]
   [junit4] Suite: org.apache.lucene.replicator.IndexAndTaxonomyReplicationClientTest
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {$full_path$=PostingsFormat(name=Asserting), $payloads$=BlockTreeOrds(blocksize=128), $facets=PostingsFormat(name=MockRandom)}, docValues:{$facets=DocValuesFormat(name=Lucene50)}, sim=DefaultSimilarity, locale=hu_HU, timezone=America/Scoresbysund
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=197578352,total=245366784
   [junit4]   2> NOTE: All tests run in this JVM: [SessionTokenTest, IndexAndTaxonomyReplicationClientTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=IndexAndTaxonomyReplicationClientTest -Dtests.seed=10CCB411FDA8B966 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=hu_HU -Dtests.timezone=America/Scoresbysund -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   0.00s J2 | IndexAndTaxonomyReplicationClientTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.RuntimeException: file handle leaks: [FileChannel(/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/replicator/test/J2/temp/lucene.replicator.IndexAndTaxonomyReplicationClientTest 10CCB411FDA8B966-001/index-MMapDirectory-001/_1j_MockRandom_0.tio)]
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([10CCB411FDA8B966]:0)
   [junit4]    > 	at org.apache.lucene.mockfile.LeakFS.onClose(LeakFS.java:64)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:78)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:79)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:79)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystem.close(FilterFileSystem.java:79)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: java.lang.Exception
   [junit4]    > 	at org.apache.lucene.mockfile.LeakFS.onOpen(LeakFS.java:47)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.callOpenHook(HandleTrackingFS.java:84)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:191)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:204)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:163)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:204)
   [junit4]    > 	at org.apache.lucene.mockfile.HandleTrackingFS.newFileChannel(HandleTrackingFS.java:163)
   [junit4]    > 	at org.apache.lucene.mockfile.FilterFileSystemProvider.newFileChannel(FilterFileSystemProvider.java:204)
   [junit4]    > 	at java.nio.channels.FileChannel.open(FileChannel.java:287)
   [junit4]    > 	at java.nio.channels.FileChannel.open(FileChannel.java:334)
   [junit4]    > 	at org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:213)
   [junit4]    > 	at org.apache.lucene.codecs.blocktreeords.OrdsBlockTreeTermsReader.<init>(OrdsBlockTreeTermsReader.java:71)
   [junit4]    > 	at org.apache.lucene.codecs.mockrandom.MockRandomPostingsFormat.fieldsProducer(MockRandomPostingsFormat.java:383)
   [junit4]    > 	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsReader.<init>(PerFieldPostingsFormat.java:258)
   [junit4]    > 	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat.fieldsProducer(PerFieldPostingsFormat.java:336)
   [junit4]    > 	at org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:104)
   [junit4]    > 	at org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:65)
   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader$1.doBody(StandardDirectoryReader.java:58)
   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader$1.doBody(StandardDirectoryReader.java:50)
   [junit4]    > 	at org.apache.lucene.index.SegmentInfos$FindSegmentsFile.run(SegmentInfos.java:660)
   [junit4]    > 	at org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:50)
   [junit4]    > 	at org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:63)
   [junit4]    > 	at org.apache.lucene.replicator.IndexAndTaxonomyReplicationClientTest.assertHandlerRevision(IndexAndTaxonomyReplicationClientTest.java:158)
   [junit4]    > 	at org.apache.lucene.replicator.IndexAndTaxonomyReplicationClientTest.testConsistencyOnExceptions(IndexAndTaxonomyReplicationClientTest.java:477)
   [junit4]    > 	... 8 more
   [junit4] Completed on J2 in 42.96s, 5 tests, 1 error <<< FAILURES!

[...truncated 12 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:456: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:2166: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/module-build.xml:58: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 7 suites, 31 tests, 1 suite-level error

Total time: 104 minutes 58 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 109219 bytes
Compression is 0.0%
Took 52 ms
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 802 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/802/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:49688/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:49688/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([F39736AAE6DD2E4F:7BC30970482143B7]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([F39736AAE6DD2E4F:7BC30970482143B7]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([F39736AAE6DD2E4F:7BC30970482143B7]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 10096 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/init-core-data-001
   [junit4]   2> 739089 T5451 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 739095 T5451 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 739096 T5452 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 739196 T5451 oasc.ZkTestServer.run start zk server on port:32363
   [junit4]   2> 739197 T5451 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 739198 T5451 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 739208 T5459 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@37b07e9a name:ZooKeeperConnection Watcher:127.0.0.1:32363 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 739208 T5451 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 739209 T5451 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 739209 T5451 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 739225 T5451 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 739226 T5451 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 739227 T5462 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1370da49 name:ZooKeeperConnection Watcher:127.0.0.1:32363/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 739227 T5451 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 739227 T5451 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 739228 T5451 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 739233 T5451 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 739234 T5451 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 739235 T5451 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 739236 T5451 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 739239 T5451 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 739241 T5451 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 739242 T5451 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 739344 T5451 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 739344 T5451 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 739346 T5451 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 739346 T5451 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 739348 T5451 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 739348 T5451 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 739349 T5451 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 739350 T5451 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 739351 T5451 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 739352 T5451 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 739353 T5451 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 739353 T5451 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 739354 T5451 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 739355 T5451 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 739356 T5451 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 739357 T5451 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 739358 T5451 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 739358 T5451 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 741082 T5451 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/control-001/cores/collection1
   [junit4]   2> 741087 T5451 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 741102 T5451 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:32371
   [junit4]   2> 741102 T5451 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/tempDir-001/control/data, hostPort=32370, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/control-001/cores, hostContext=/}
   [junit4]   2> 741103 T5451 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 741103 T5451 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/control-001/'
   [junit4]   2> 741163 T5451 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/control-001/solr.xml
   [junit4]   2> 741231 T5451 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/control-001/cores
   [junit4]   2> 741232 T5451 oasc.CoreContainer.<init> New CoreContainer 989665152
   [junit4]   2> 741232 T5451 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/control-001/]
   [junit4]   2> 741232 T5451 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/control-001/lib
   [junit4]   2> 741233 T5451 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/control-001/lib).
   [junit4]   2> 741244 T5451 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 741244 T5451 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 741244 T5451 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 741245 T5451 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 741245 T5451 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 741245 T5451 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 741245 T5451 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 741246 T5451 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 741246 T5451 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 741246 T5451 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 741246 T5451 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 741247 T5451 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 741247 T5451 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 741247 T5451 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 741247 T5451 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 741247 T5451 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:32363/solr
   [junit4]   2> 741250 T5451 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 741250 T5451 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 741251 T5451 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 741252 T5474 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5ad3e6de name:ZooKeeperConnection Watcher:127.0.0.1:32363 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 741254 T5451 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 741255 T5451 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 741262 T5451 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 741263 T5477 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@19f95270 name:ZooKeeperConnection Watcher:127.0.0.1:32363/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 741263 T5451 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 741268 T5451 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 741270 T5451 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 741271 T5451 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 741272 T5451 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 741274 T5451 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 741275 T5451 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 741276 T5451 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 741277 T5451 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 741278 T5451 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32370_
   [junit4]   2> 741278 T5451 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32370_
   [junit4]   2> 741279 T5451 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 741280 T5451 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 741281 T5451 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 741282 T5451 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:32370_
   [junit4]   2> 741282 T5451 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 741283 T5451 oasc.Overseer.start Overseer (id=93567674461716483-127.0.0.1:32370_-n_0000000000) starting
   [junit4]   2> 741284 T5451 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 741288 T5451 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 741288 T5479 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 741288 T5451 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 741289 T5478 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 741294 T5451 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/control-001/cores
   [junit4]   2> 741296 T5451 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, config=solrconfig.xml, loadOnStartup=true, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/control-001/cores/collection1/, schema=schema.xml, shard=, name=collection1, dataDir=data/, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/control-001/cores/collection1, collection=control_collection}
   [junit4]   2> 741296 T5451 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/control-001/cores/collection1/
   [junit4]   2> 741296 T5451 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 741297 T5481 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 741297 T5481 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 741298 T5481 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 741298 T5477 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 741298 T5478 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:32370_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32370",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 741299 T5478 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:32370_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32370",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 741299 T5478 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 741301 T5478 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 741302 T5477 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 742298 T5481 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 742299 T5481 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 742299 T5481 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 742300 T5481 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 742300 T5481 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 742300 T5481 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/control-001/cores/collection1/'
   [junit4]   2> 742323 T5481 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 742325 T5481 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 742335 T5481 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 742357 T5481 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 742404 T5481 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 742505 T5481 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 742514 T5481 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 742909 T5481 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 742924 T5481 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 742927 T5481 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 742936 T5481 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 742940 T5481 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 742943 T5481 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 742944 T5481 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 742944 T5481 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 742944 T5481 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 742945 T5481 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 742945 T5481 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 742945 T5481 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 742946 T5481 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 742946 T5481 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/control-001/cores/collection1/data/
   [junit4]   2> 742946 T5481 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a7d758a
   [junit4]   2> 742947 T5481 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/control-001/cores/collection1/data
   [junit4]   2> 742947 T5481 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/control-001/cores/collection1/data/index/
   [junit4]   2> 742947 T5481 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 742948 T5481 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/control-001/cores/collection1/data/index
   [junit4]   2> 742948 T5481 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=7, maxMergeAtOnceExplicit=3, maxMergedSegmentMB=0.8154296875, floorSegmentMB=1.2998046875, forceMergeDeletesPctAllowed=9.672109035611063, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 742949 T5481 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@7d6c5bc7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@54fda625; maxCacheMB=0.7509765625 maxMergeSizeMB=0.17578125)),segFN=segments_1,generation=1}
   [junit4]   2> 742949 T5481 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 742954 T5481 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 742954 T5481 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 742955 T5481 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 742955 T5481 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 742955 T5481 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 742955 T5481 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 742956 T5481 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 742956 T5481 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 742956 T5481 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 742975 T5481 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 742977 T5481 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 742979 T5481 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 742981 T5481 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 742986 T5481 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/segments,standard,/admin/luke,/admin/plugins,/replication,/update/json,/admin/system,/update/json/docs,/admin/file,/schema,/config,/update,/admin/ping,/admin/mbeans,/admin/properties,/update/csv,/admin/threads,/admin/logging,/get
   [junit4]   2> 742988 T5481 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 742989 T5481 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 742989 T5481 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 742991 T5481 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 742991 T5481 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 742992 T5481 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=5, maxMergeAtOnceExplicit=8, maxMergedSegmentMB=27.9404296875, floorSegmentMB=1.2099609375, forceMergeDeletesPctAllowed=2.72183618406439, segmentsPerTier=33.0, maxCFSSegmentSizeMB=1.63671875, noCFSRatio=1.0
   [junit4]   2> 742992 T5481 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@7d6c5bc7 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@54fda625; maxCacheMB=0.7509765625 maxMergeSizeMB=0.17578125)),segFN=segments_1,generation=1}
   [junit4]   2> 742993 T5481 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 742993 T5481 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@367e0e18[collection1] main
   [junit4]   2> 742993 T5481 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 742994 T5481 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 742994 T5481 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 742995 T5481 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 742995 T5481 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 742995 T5481 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 742996 T5481 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 742996 T5481 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 742996 T5481 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 742996 T5481 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 742997 T5481 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 742997 T5482 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@367e0e18[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 742997 T5485 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:32370 collection:control_collection shard:shard1
   [junit4]   2> 742998 T5451 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 742998 T5451 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 742998 T5485 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 743001 T5451 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 743001 T5451 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 743001 T5485 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 743002 T5489 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5c4cc063 name:ZooKeeperConnection Watcher:127.0.0.1:32363/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 743003 T5451 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 743003 T5477 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 743003 T5451 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 743003 T5485 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 743003 T5451 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 743003 T5485 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 743003 T5478 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C2368 name=collection1 org.apache.solr.core.SolrCore@56509d3a url=http://127.0.0.1:32370/collection1 node=127.0.0.1:32370_ C2368_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:32370_, base_url=http://127.0.0.1:32370, core=collection1, state=down}
   [junit4]   2> 743009 T5485 C:control_collection S:shard1 C2368 P32370 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:32370/collection1/
   [junit4]   2> 743009 T5485 C:control_collection S:shard1 C2368 P32370 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 743010 T5485 C:control_collection S:shard1 C2368 P32370 oasc.SyncStrategy.syncToMe http://127.0.0.1:32370/collection1/ has no replicas
   [junit4]   2> 743010 T5485 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:32370/collection1/ shard1
   [junit4]   2> 743010 T5485 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 743010 T5489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 743010 T5477 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 743013 T5451 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 743017 T5477 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 743018 T5478 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32370",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 743121 T5477 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 743121 T5489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 743170 T5485 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:32370/collection1/ and leader is http://127.0.0.1:32370/collection1/
   [junit4]   2> 743171 T5485 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:32370
   [junit4]   2> 743171 T5485 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 743171 T5485 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 743171 T5485 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 743172 T5477 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 743173 T5478 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:32370_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32370",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1"} current state version: 3
   [junit4]   2> 743173 T5478 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:32370_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32370",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 743276 T5477 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 743276 T5489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 744564 T5451 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/shard-1-001/cores/collection1
   [junit4]   2> 744567 T5451 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/shard-1-001
   [junit4]   2> 744568 T5451 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 744575 T5451 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:32388
   [junit4]   2> 744575 T5451 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=32387, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/shard-1-001/cores, hostContext=/, solrconfig=solrconfig.xml}
   [junit4]   2> 744575 T5451 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 744576 T5451 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/shard-1-001/'
   [junit4]   2> 744623 T5451 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/shard-1-001/solr.xml
   [junit4]   2> 744679 T5451 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/shard-1-001/cores
   [junit4]   2> 744679 T5451 oasc.CoreContainer.<init> New CoreContainer 2109986782
   [junit4]   2> 744679 T5451 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/shard-1-001/]
   [junit4]   2> 744680 T5451 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/shard-1-001/lib
   [junit4]   2> 744681 T5451 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/shard-1-001/lib).
   [junit4]   2> 744694 T5451 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 744694 T5451 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 744695 T5451 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 744695 T5451 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 744695 T5451 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 744696 T5451 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 744696 T5451 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 744696 T5451 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 744697 T5451 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 744697 T5451 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 744697 T5451 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 744698 T5451 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 744698 T5451 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 744698 T5451 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 744699 T5451 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 744699 T5451 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:32363/solr
   [junit4]   2> 744699 T5451 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 744700 T5451 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 744700 T5451 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 744702 T5501 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1e1d5865 name:ZooKeeperConnection Watcher:127.0.0.1:32363 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 744704 T5451 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 744705 T5451 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 744713 T5451 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 744714 T5504 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7102671b name:ZooKeeperConnection Watcher:127.0.0.1:32363/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 744714 T5451 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 744722 T5451 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 745740 T5451 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32387_
   [junit4]   2> 745748 T5451 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32387_
   [junit4]   2> 745753 T5451 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 745755 T5451 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/shard-1-001/cores
   [junit4]   2> 745756 T5451 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, collection=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/shard-1-001/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/shard-1-001/cores/collection1/, schema=schema.xml, transient=false, coreNodeName=, dataDir=data/, loadOnStartup=true, shard=, name=collection1}
   [junit4]   2> 745757 T5451 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/shard-1-001/cores/collection1/
   [junit4]   2> 745757 T5451 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 745758 T5505 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 745758 T5505 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 745759 T5505 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 745759 T5477 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 745760 T5478 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:32387_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32387",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null} current state version: 4
   [junit4]   2> 745760 T5478 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:32387_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32387",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 745760 T5478 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 745760 T5478 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 745761 T5489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 745761 T5504 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 745761 T5477 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 746759 T5505 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 746760 T5505 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 746760 T5505 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 746761 T5505 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 746761 T5505 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 746761 T5505 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/shard-1-001/cores/collection1/'
   [junit4]   2> 746783 T5505 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 746785 T5505 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 746795 T5505 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 746817 T5505 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 746879 T5505 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 746980 T5505 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 746990 T5505 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 747355 T5505 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 747365 T5505 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 747367 T5505 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 747374 T5505 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 747377 T5505 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 747379 T5505 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 747380 T5505 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 747380 T5505 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 747380 T5505 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 747381 T5505 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 747381 T5505 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 747381 T5505 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 747381 T5505 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 747382 T5505 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 747382 T5505 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3a7d758a
   [junit4]   2> 747382 T5505 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/shard-1-001/cores/collection1/data
   [junit4]   2> 747383 T5505 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 747383 T5505 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 747383 T5505 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F39736AAE6DD2E4F-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 747383 T5505 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=7, maxMergeAtOnceExplicit=3, maxMergedSegmentMB=0.8154296875, floorSegmentMB=1.2998046875, forceMergeDeletesPctAllowed=9.672109035611063, segmentsPerTier=11.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 747384 T5505 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@6bade44e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6567f41c; maxCacheMB=0.7509765625 maxMergeSizeMB=0.17578125)),segFN=segments_1,generation=1}
   [junit4]   2> 747384 T5505 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 747388 T5505 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 747388 T5505 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 747389 T5505 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 747389 T5505 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 747389 T5505 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 747389 T5505 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 747390 T5505 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 747390 T5505 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 747390 T5505 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 747406 T5505 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 747407 T5505 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 747409 T5505 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 747411 T5505 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 747415 T5505 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/segments,standard,/admin/luke,/admin/plugins,/replication,/update/json,/admin/system,/update/json/docs,/admin/file,/schema,/config,/update,/admin/ping,/admin/mbeans,/admin/properties,/update/csv,/admin/threads,/admin/logging,/get
   [junit4]   2> 747417 T5505 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 747418 T5505 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 747418 T5505 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 747419 T5505 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 747419 T5505 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 747420 T5505 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=5, maxMergeAtOnceExplicit=8, maxMergedSegmentMB=27.9404296875, floorSegmentMB=1.2099609375, forceMergeDeletesPctAllowed=2.72183618406439, segmentsPerTier=33.0, maxCFSSegmentSizeMB=1.63671875, noCFSRatio=1.0
   [junit4]   2> 747420 T5505 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@6bade44e lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6567f41c; maxCacheMB=0.7509765625 maxMergeSizeMB=0.17578125)),segFN=segments_1,generation=1}
   [junit4]   2> 747420 T5505 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 747420 T5505 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@59fb876[collection1] main
   [junit4]   2> 747421 T5505 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 747421 T5505 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 747421 T5505 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 747422 T5505 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 747422 T5505 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 747422 T5505 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 747422 T5505 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 747422 T5505 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 747423 T5505 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 747423 T5505 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 747423 T5506 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@59fb876[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 747424 T5505 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 747424 T5509 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:32387 collection:collection1 shard:shard1
   [junit4]   2> 747425 T5451 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 747425 T5451 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 747425 T5509 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 747428 T5509 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 747429 T5477 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 747429 T5509 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 747430 T5509 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C2369 name=collection1 org.apache.solr.core.SolrCore@446befb0 url=http://127.0.0.1:32387/collection1 node=127.0.0.1:32387_ C2369_STATE=coll:collection1 core:collection1 props:{node_name=127.0.0.1:32387_, base_url=http://127.0.0.1:32387, core=collection1, state=down}
   [junit4]   2> 747430 T5509 C:collection1 S:shard1 C2369 P32387 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:32387/collection1/
   [junit4]   2> 747430 T5478 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 747430 T5509 C:collection1 S:shard1 C2369 P32387 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 747431 T5509 C:collection1 S:shard1 C2369 P32387 oasc.SyncStrategy.syncToMe http://127.0.0.1:32387/collection1/ has no replicas
   [junit4]   2> 747431 T5509 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:32387/collection1/ shard1
   [junit4]   2> 747431 T5509 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 747431 T5477 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 747432 T5489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 747432 T5504 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 747434 T5477 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 747435 T5478 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32387",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 747538 T5489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 747538 T5477 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 747538 T5504 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 747587 T5509 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:32387/collection1/ and leader is http://127.0.0.1:32387/collection1/
   [junit4]   2> 747588 T5509 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:32387
   [junit4]   2> 747588 T5509 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 747588 T5509 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 747588 T5509 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 747589 T5477 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 747589 T5478 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:32387_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32387",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1"} current state version: 7
   [junit4]   2> 747590 T5478 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:32387_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32387",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 747693 T5489 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 747693 T5504 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 747693 T5477 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate

[...truncated too long message...]

=it -Dtests.timezone=PLT -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   80.6s J0 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F39736AAE6DD2E4F:7BC30970482143B7]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 2736365 T10474 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest F39736AAE6DD2E4F-001
   [junit4]   2> 80576 T10473 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {a_t=FSTOrd50, range_facet_l=FSTOrd50, text=PostingsFormat(name=MockRandom), timestamp=FSTOrd50, multiDefault=FSTOrd50, id=PostingsFormat(name=SimpleText), intDefault=FST50, _version_=FST50}, docValues:{timestamp=DocValuesFormat(name=Direct)}, sim=DefaultSimilarity, locale=it, timezone=PLT
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=114705144,total=415760384
   [junit4]   2> NOTE: All tests run in this JVM: [HttpPartitionTest, ActionThrottleTest, AddSchemaFieldsUpdateProcessorFactoryTest, AsyncMigrateRouteKeyTest, TestFieldTypeCollectionResource, TestSolrJ, SpellCheckCollatorTest, TestExactStatsCache, XmlUpdateRequestHandlerTest, TestMiniSolrCloudClusterSSL, UUIDFieldTest, SolrIndexConfigTest, TestUpdate, TestAtomicUpdateErrorCases, StatelessScriptUpdateProcessorFactoryTest, OverseerRolesTest, TestRawResponseWriter, TestXIncludeConfig, RequestLoggingTest, TestCopyFieldCollectionResource, TestStandardQParsers, TestRestManager, ShardRoutingCustomTest, SignatureUpdateProcessorFactoryTest, ZkNodePropsTest, TestFileDictionaryLookup, SuggesterTest, TimeZoneUtilsTest, TestFreeTextSuggestions, TestDynamicFieldResource, DistributedQueryElevationComponentTest, OpenExchangeRatesOrgProviderTest, DeleteReplicaTest, TestFastLRUCache, DistributedSpellCheckComponentTest, AnalyticsMergeStrategyTest, SimpleCollectionCreateDeleteTest, ShowFileRequestHandlerTest, BasicFunctionalityTest, DirectSolrSpellCheckerTest, ExternalFileFieldSortTest, DocExpirationUpdateProcessorFactoryTest, TestDocSet, TestRandomFaceting, TestManagedResourceStorage, RegexBoostProcessorTest, TestOrdValues, TestTrie, SearchHandlerTest, TestCloudManagedSchema, ZkStateWriterTest, TestShortCircuitedRequests, TestCollapseQParserPlugin, TestSchemaResource, OverseerTest, TestSolrCoreProperties, ConnectionManagerTest, TestCollationField, TestIBSimilarityFactory, OpenCloseCoreStressTest, AutoCommitTest, TestDistributedGrouping, TestFaceting, ZkControllerTest, TestStressReorder, TestReload, TestRangeQuery, TestCoreContainer, SimpleFacetsTest, TestSolr4Spatial, SpellCheckComponentTest, SolrCmdDistributorTest, QueryElevationComponentTest, BadIndexSchemaTest, TestSort, TestLazyCores, OverseerCollectionProcessorTest, SimplePostToolTest, TestCoreDiscovery, SuggesterTSTTest, TestStressLucene, SuggesterWFSTTest, TestPseudoReturnFields, QueryEqualityTest, SolrInfoMBeanTest, DebugComponentTest, IndexBasedSpellCheckerTest, StandardRequestHandlerTest, TestQueryTypes, TestOmitPositions, RequestHandlersTest, PathHierarchyTokenizerFactoryTest, RequiredFieldsTest, TestArbitraryIndexDir, IndexSchemaTest, ReturnFieldsTest, TestCSVResponseWriter, QueryParsingTest, JsonLoaderTest, TestPHPSerializedResponseWriter, SOLR749Test, UpdateParamsTest, TestSolrIndexConfig, ResponseLogComponentTest, TestSolrDeletionPolicy2, MultiTermTest, TestSearchPerf, MinimalSchemaTest, TestElisionMultitermQuery, TestPostingsSolrHighlighter, DirectSolrConnectionTest, TestLuceneMatchVersion, TestPhraseSuggestions, SynonymTokenizerTest, EchoParamsTest, TestPerFieldSimilarity, TestDefaultSimilarityFactory, ResourceLoaderTest, PreAnalyzedFieldTest, TestSystemIdResolver, PrimUtilsTest, TestSuggestSpellingConverter, SpellingQueryConverterTest, DOMUtilTest, RAMDirectoryFactoryTest, ClusterStateTest, TestDocumentBuilder, DistributedMLTComponentTest, TestRTGBase, DistributedIntervalFacetingTest, TestTolerantSearch, DeleteInactiveReplicaTest, LeaderInitiatedRecoveryOnCommitTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverseerStatusTest, ReplicationFactorTest]
   [junit4] Completed on J0 in 81.10s, 1 test, 1 error <<< FAILURES!

[...truncated 255 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 58 ignored (9 assumptions)

Total time: 153 minutes 11 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 856731391 bytes
Compression is 0.0%
Took 2 min 33 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 801 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/801/

4 tests failed.
REGRESSION:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
Invalid content type: 

Stack Trace:
org.apache.http.ParseException: Invalid content type: 
	at __randomizedtesting.SeedInfo.seed([12B7CA3DDFF6AD2:897F43797303072A]:0)
	at org.apache.http.entity.ContentType.parse(ContentType.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:509)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:224)
	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:166)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:671)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test(FullSolrCloudDistribCmdsTest.java:151)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:51418/m_zvk/j/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51418/m_zvk/j/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([12B7CA3DDFF6AD2:897F43797303072A]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([12B7CA3DDFF6AD2:897F43797303072A]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([12B7CA3DDFF6AD2:897F43797303072A]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 10019 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/init-core-data-001
   [junit4]   2> 191581 T329 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 191585 T329 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 191586 T330 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 191686 T329 oasc.ZkTestServer.run start zk server on port:21283
   [junit4]   2> 191687 T329 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 191688 T329 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 191693 T337 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@443cafb6 name:ZooKeeperConnection Watcher:127.0.0.1:21283 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 191694 T329 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 191695 T329 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 191695 T329 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 191698 T329 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 191699 T329 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 191701 T340 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7f584177 name:ZooKeeperConnection Watcher:127.0.0.1:21283/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 191702 T329 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 191702 T329 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 191703 T329 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 191706 T329 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 191710 T329 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 191712 T329 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 191716 T329 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 191716 T329 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 191721 T329 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 191722 T329 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 191726 T329 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 191727 T329 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 191730 T329 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 191731 T329 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 191734 T329 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 191734 T329 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 191738 T329 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 191738 T329 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 191741 T329 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 191742 T329 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 191746 T329 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 191746 T329 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 191749 T329 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 191750 T329 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 191753 T329 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 191754 T329 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 191757 T329 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 191758 T329 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 193512 T329 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/control-001/cores/collection1
   [junit4]   2> 193517 T329 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 193527 T329 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:21291
   [junit4]   2> 193527 T329 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/tempDir-001/control/data, hostPort=21290, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/control-001/cores}
   [junit4]   2> 193528 T329 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 193528 T329 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/control-001/'
   [junit4]   2> 193578 T329 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/control-001/solr.xml
   [junit4]   2> 193642 T329 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/control-001/cores
   [junit4]   2> 193643 T329 oasc.CoreContainer.<init> New CoreContainer 1926744539
   [junit4]   2> 193643 T329 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/control-001/]
   [junit4]   2> 193644 T329 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/control-001/lib
   [junit4]   2> 193645 T329 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/control-001/lib).
   [junit4]   2> 193660 T329 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 193661 T329 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 193661 T329 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 193662 T329 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 193662 T329 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 193663 T329 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 193663 T329 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 193663 T329 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 193664 T329 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 193664 T329 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 193665 T329 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 193666 T329 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 193666 T329 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 193667 T329 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 193667 T329 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 193667 T329 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:21283/solr
   [junit4]   2> 193668 T329 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 193668 T329 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 193669 T329 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 193672 T352 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4c6b4dfd name:ZooKeeperConnection Watcher:127.0.0.1:21283 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 193672 T329 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 193674 T329 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 193678 T329 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 193681 T355 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5be463b5 name:ZooKeeperConnection Watcher:127.0.0.1:21283/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 193681 T329 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 193684 T329 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 193689 T329 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 193694 T329 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 193699 T329 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 193703 T329 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 193709 T329 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 193712 T329 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 193716 T329 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 193718 T329 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:21290_
   [junit4]   2> 193720 T329 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:21290_
   [junit4]   2> 193724 T329 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 193728 T329 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 193731 T329 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 193734 T329 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:21290_
   [junit4]   2> 193734 T329 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 193738 T329 oasc.Overseer.start Overseer (id=93563027219546115-127.0.0.1:21290_-n_0000000000) starting
   [junit4]   2> 193742 T329 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 193757 T329 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 193758 T357 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 193758 T329 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 193759 T356 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 193767 T329 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/control-001/cores
   [junit4]   2> 193770 T329 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/control-001/cores/collection1/, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/control-001/cores/collection1, schema=schema.xml, dataDir=data/, transient=false, shard=, name=collection1, coreNodeName=, collection=control_collection, config=solrconfig.xml}
   [junit4]   2> 193771 T329 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/control-001/cores/collection1/
   [junit4]   2> 193771 T329 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 193773 T359 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 193774 T359 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 193775 T355 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 193775 T359 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 193777 T356 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:21290",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:21290_",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 0
   [junit4]   2> 193779 T356 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:21290",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:21290_",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 193779 T356 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 193779 T356 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 193782 T355 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 194776 T359 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 194777 T359 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 194778 T359 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 194779 T359 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 194780 T359 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 194780 T359 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/control-001/cores/collection1/'
   [junit4]   2> 194802 T359 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 194806 T359 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 194818 T359 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 194858 T359 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 194940 T359 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 194942 T359 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 194960 T359 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 195457 T359 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 195473 T359 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 195476 T359 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 195499 T359 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 195504 T359 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 195510 T359 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 195512 T359 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 195512 T359 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 195512 T359 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 195514 T359 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 195514 T359 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 195514 T359 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 195514 T359 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 195515 T359 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/control-001/cores/collection1/data/
   [junit4]   2> 195515 T359 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@693e20a9
   [junit4]   2> 195516 T359 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/control-001/cores/collection1/data
   [junit4]   2> 195516 T359 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/control-001/cores/collection1/data/index/
   [junit4]   2> 195516 T359 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 195517 T359 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/control-001/cores/collection1/data/index
   [junit4]   2> 195517 T359 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=6, maxMergeAtOnceExplicit=2, maxMergedSegmentMB=88.2705078125, floorSegmentMB=1.2978515625, forceMergeDeletesPctAllowed=21.22391057653146, segmentsPerTier=8.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 195518 T359 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@4403a302 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4ac2fb30),segFN=segments_1,generation=1}
   [junit4]   2> 195518 T359 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 195524 T359 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 195524 T359 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 195524 T359 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 195525 T359 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 195525 T359 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 195525 T359 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 195525 T359 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 195526 T359 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 195526 T359 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 195542 T359 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 195544 T359 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 195546 T359 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 195548 T359 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 195554 T359 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /replication,/update/csv,/config,/admin/luke,/schema,/admin/properties,/admin/segments,/admin/file,/admin/plugins,/update,/admin/mbeans,/admin/logging,/get,/admin/system,/admin/threads,standard,/update/json,/update/json/docs,/admin/ping
   [junit4]   2> 195556 T359 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 195559 T359 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 195560 T359 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 195562 T359 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 195562 T359 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 195563 T359 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=42, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=71.466796875, floorSegmentMB=1.7568359375, forceMergeDeletesPctAllowed=28.46187945139831, segmentsPerTier=18.0, maxCFSSegmentSizeMB=0.923828125, noCFSRatio=0.6285774910282181
   [junit4]   2> 195563 T359 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@4403a302 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4ac2fb30),segFN=segments_1,generation=1}
   [junit4]   2> 195564 T359 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 195564 T359 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@3c8d340f[collection1] main
   [junit4]   2> 195564 T359 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 195566 T359 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 195567 T359 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 195567 T359 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 195568 T359 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 195568 T359 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 195569 T359 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 195569 T359 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 195570 T359 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 195570 T359 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 195571 T360 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3c8d340f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 195572 T359 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 195573 T363 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:21290 collection:control_collection shard:shard1
   [junit4]   2> 195573 T329 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 195574 T329 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 195575 T363 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 195578 T329 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 195579 T329 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 195581 T367 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7796dfcc name:ZooKeeperConnection Watcher:127.0.0.1:21283/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 195581 T329 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 195582 T329 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 195582 T329 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 195585 T363 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 195588 T355 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 195589 T363 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 195589 T363 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C100 name=collection1 org.apache.solr.core.SolrCore@62d8ef4f url=http://127.0.0.1:21290/collection1 node=127.0.0.1:21290_ C100_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, base_url=http://127.0.0.1:21290, node_name=127.0.0.1:21290_}
   [junit4]   2> 195589 T363 C:control_collection S:shard1 C100 P21290 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:21290/collection1/
   [junit4]   2> 195590 T329 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 195590 T356 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 195590 T363 C:control_collection S:shard1 C100 P21290 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 195590 T329 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 195591 T363 C:control_collection S:shard1 C100 P21290 oasc.SyncStrategy.syncToMe http://127.0.0.1:21290/collection1/ has no replicas
   [junit4]   2> 195591 T329 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 195591 T363 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:21290/collection1/ shard1
   [junit4]   2> 195592 T363 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 195592 T329 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 195593 T355 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 195593 T367 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 195595 T370 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@18b8111c name:ZooKeeperConnection Watcher:127.0.0.1:21283/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 195595 T329 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 195596 T329 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 195604 T356 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 195604 T356 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 195605 T355 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 195611 T356 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:21290",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 195612 T356 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 195614 T355 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 195614 T367 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 195621 T355 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 195623 T355 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 195623 T367 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 195658 T363 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:21290/collection1/ and leader is http://127.0.0.1:21290/collection1/
   [junit4]   2> 195658 T363 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:21290
   [junit4]   2> 195658 T363 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 195659 T363 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 195659 T363 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 195662 T355 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 195664 T356 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:21290",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:21290_",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 4
   [junit4]   2> 195665 T356 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:21290",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:21290_",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 195772 T367 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 195772 T355 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 197275 T329 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/shard-1-001/cores/collection1
   [junit4]   2> 197279 T329 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/shard-1-001
   [junit4]   2> 197280 T329 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 197283 T329 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:21297
   [junit4]   2> 197283 T329 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=21296, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/shard-1-001/cores}
   [junit4]   2> 197284 T329 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 197285 T329 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/shard-1-001/'
   [junit4]   2> 197336 T329 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/shard-1-001/solr.xml
   [junit4]   2> 197395 T329 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/shard-1-001/cores
   [junit4]   2> 197395 T329 oasc.CoreContainer.<init> New CoreContainer 1378699221
   [junit4]   2> 197396 T329 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/shard-1-001/]
   [junit4]   2> 197396 T329 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/shard-1-001/lib
   [junit4]   2> 197397 T329 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/shard-1-001/lib).
   [junit4]   2> 197411 T329 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 197411 T329 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 197412 T329 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 197412 T329 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 197412 T329 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 197413 T329 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 197413 T329 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 197413 T329 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 197414 T329 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 197414 T329 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 197414 T329 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 197415 T329 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 197415 T329 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 197416 T329 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 197416 T329 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 197416 T329 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:21283/solr
   [junit4]   2> 197417 T329 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 197417 T329 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 197418 T329 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 197421 T382 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7d72596b name:ZooKeeperConnection Watcher:127.0.0.1:21283 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 197421 T329 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 197422 T329 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 197425 T329 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 197428 T385 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@45bc2b59 name:ZooKeeperConnection Watcher:127.0.0.1:21283/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 197428 T329 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 197438 T329 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 198451 T329 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:21296_
   [junit4]   2> 198452 T329 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:21296_
   [junit4]   2> 198457 T329 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 198461 T329 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/shard-1-001/cores
   [junit4]   2> 198464 T329 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/shard-1-001/cores/collection1/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/shard-1-001/cores/collection1, coreNodeName=, name=collection1, loadOnStartup=true, shard=, dataDir=data/, config=solrconfig.xml, schema=schema.xml, collection=collection1}
   [junit4]   2> 198465 T329 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/shard-1-001/cores/collection1/
   [junit4]   2> 198465 T329 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 198469 T386 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 198470 T386 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 198473 T355 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 198474 T386 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 198474 T356 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:21296",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:21296_",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 198474 T386 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 198475 T356 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:21296",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:21296_",
   [junit4]   2> 	  "collection":"collection1"}
   [junit4]   2> 198476 T386 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 198476 T356 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 198476 T356 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 198479 T386 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 198479 T386 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 198479 T356 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 198480 T385 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 198482 T385 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 199479 T386 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 199480 T386 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 199481 T386 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 199482 T386 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 199483 T386 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 199483 T386 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/shard-1-001/cores/collection1/'
   [junit4]   2> 199506 T386 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 199510 T386 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 199527 T386 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 199555 T386 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 199618 T386 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 199620 T386 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 199638 T386 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 200188 T386 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 200211 T386 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 200215 T386 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 200243 T386 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 200249 T386 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 200256 T386 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 200258 T386 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 200258 T386 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 200259 T386 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 200260 T386 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 200260 T386 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 200261 T386 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 200261 T386 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 200261 T386 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 200262 T386 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@693e20a9
   [junit4]   2> 200263 T386 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/shard-1-001/cores/collection1/data
   [junit4]   2> 200263 T386 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 200263 T386 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 200264 T386 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 12B7CA3DDFF6AD2-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 200264 T386 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=6, maxMergeAtOnceExplicit=2, maxMergedSegmentMB=88.2705078125, floorSegmentMB=1.2978515625, forceMergeDeletesPctAllowed=21.22391057653146, segmentsPerTier=8.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 200265 T386 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@6b2ef5bf lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@58080592),segFN=segments_1,generation=1}
   [junit4]   2> 200266 T386 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 200274 T386 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 200275 T386 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 200276 T386 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 200276 T386 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 200276 T386 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 200277 T386 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 200277 T386 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 200278 T386 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 200278 T386 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 200299 T386 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 200303 T386 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 200307 T386 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 200310 T386 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 200319 T386 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /replication,/update/csv,/config,/admin/luke,/schema,/admin/properties,/admin/segments,/admin/file,/admin/plugins,/update,/admin/mbeans,/admin/logging,/get,/admin/system,/admin/threads,standard,/update/json,/update/json/docs,/admin/ping
   [junit4]   2> 200322 T386 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 200324 T386 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 200325 T386 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 200327 T386 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 200327 T386 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 200329 T386 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=42, maxMergeAtOnceExplicit=28, maxMergedSegmentMB=71.466796875, floorSegmentMB=1.7568359375, forceMergeDeletesPctAllowed=28.46187945139831, segmentsPerTier=18.0, maxCFSSegmentSizeMB=0.923828125, noCFSRatio=0.6285774910282181
   [junit4]   2> 200329 T386 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@6b2ef5bf lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@58080592),segFN=segments_1,generation=1}
   [junit4]   2> 200330 T386 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 200331 T386 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@5916d195[collection1] main
   [junit4]   2> 200332 T386 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 200334 T386 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 200334 T386 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 200335 T386 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 200335 T386 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 200335 T386 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 200336 T386 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 200336 T386 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 200337 T386 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 200337 T386 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 200339 T387 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5916d195[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 200339 T386 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 200340 T390 C:collection1 S:shard2 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1

[...truncated too long message...]

urrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		4	/solr/collections/collection1/state.json
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/queue
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 2462241 T36690 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51438/m_zvk/j
   [junit4]   2> 2462242 T36690 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:51418/m_zvk/j
   [junit4]   2> 2462242 T36690 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:51429/m_zvk/j
   [junit4]   2> 2462242 T36690 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:15876/m_zvk/j
   [junit4]   2> 2462243 T36690 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 10 connections to: http://127.0.0.1:62691/m_zvk/j
   [junit4]   2> 2462243 T36690 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51438/m_zvk/j
   [junit4]   2> 2462244 T36690 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51418/m_zvk/j
   [junit4]   2> 2462244 T36690 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51429/m_zvk/j
   [junit4]   2> 2462244 T36690 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:15876/m_zvk/j
   [junit4]   2> 2462244 T36690 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:62691/m_zvk/j
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=12B7CA3DDFF6AD2 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=es_MX -Dtests.timezone=Etc/GMT+12 -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   75.9s J2 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51418/m_zvk/j/c8n_1x3_commits_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([12B7CA3DDFF6AD2:897F43797303072A]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 2462267 T36690 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 12B7CA3DDFF6AD2-001
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=es_MX, timezone=Etc/GMT+12
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=201087664,total=318242816
   [junit4]   2> NOTE: All tests run in this JVM: [TestSolrQueryParser, TestRTGBase, TestExtendedDismaxParser, SOLR749Test, AnalysisErrorHandlingTest, LeaderFailoverAfterPartitionTest, TestAnalyzedSuggestions, ResponseHeaderTest, TestLRUCache, TestDocBasedVersionConstraints, DirectUpdateHandlerTest, CurrencyFieldOpenExchangeTest, ChaosMonkeyNothingIsSafeTest, TestReloadAndDeleteDocs, TestFieldSortValues, RemoteQueryErrorTest, AliasIntegrationTest, CursorMarkTest, TestComponentsName, SaslZkACLProviderTest, TestPHPSerializedResponseWriter, ReturnFieldsTest, DocumentBuilderTest, DocValuesMissingTest, TestDistributedSearch, CachingDirectoryFactoryTest, TestReqParamsAPI, SpellCheckComponentTest, ExitableDirectoryReaderTest, RankQueryTest, TestOverriddenPrefixQueryForCustomFieldType, StandardRequestHandlerTest, HardAutoCommitTest, CacheHeaderTest, TestSolrXml, TestExpandComponent, ShardSplitTest, SolrCoreCheckLockOnStartupTest, TestLuceneMatchVersion, CustomCollectionTest, RAMDirectoryFactoryTest, TestHdfsUpdateLog, ExternalCollectionsTest, CursorPagingTest, TestReload, LeaderElectionIntegrationTest, SolrIndexSplitterTest, TestChildDocTransformer, TestFoldingMultitermQuery, FileBasedSpellCheckerTest, LukeRequestHandlerTest, TestCryptoKeys, MoreLikeThisHandlerTest, DirectUpdateHandlerOptimizeTest, MigrateRouteKeyTest, CloudExitableDirectoryReaderTest, DeleteLastCustomShardedReplicaTest, DeleteReplicaTest, DistribJoinFromCollectionTest, DistributedQueueTest, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J2 in 75.94s, 1 test, 1 error <<< FAILURES!

[...truncated 279 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 4 errors, 58 ignored (9 assumptions)

Total time: 141 minutes 52 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 864980184 bytes
Compression is 0.0%
Took 2 min 30 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 800 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/800/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:56864/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:56864/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([7FF19CB2C80AC917:F7A5A36866F6A4EF]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33136/gds/e/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33136/gds/e/collection1
	at __randomizedtesting.SeedInfo.seed([7FF19CB2C80AC917:F7A5A36866F6A4EF]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33136/gds/e/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:32680/vpe/ei/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:32680/vpe/ei/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([7FF19CB2C80AC917:F7A5A36866F6A4EF]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:32680/vpe/ei/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 53 more




Build Log:
[...truncated 9950 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/init-core-data-001
   [junit4]   2> 45861 T41 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /vpe/ei
   [junit4]   2> 45919 T41 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 45931 T42 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 46334 T41 oasc.ZkTestServer.run start zk server on port:24732
   [junit4]   2> 46502 T41 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 46768 T41 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 46973 T49 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3ae02032 name:ZooKeeperConnection Watcher:127.0.0.1:24732 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 46974 T41 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 46975 T41 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 46988 T41 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 47126 T41 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 47128 T41 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 47131 T52 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3d57bdef name:ZooKeeperConnection Watcher:127.0.0.1:24732/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 47132 T41 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 47133 T41 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 47149 T41 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 47156 T41 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 47161 T41 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 47165 T41 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 47170 T41 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 47172 T41 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 47179 T41 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 47180 T41 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 47184 T41 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 47185 T41 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 47189 T41 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 47190 T41 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 47194 T41 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 47195 T41 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 47200 T41 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 47201 T41 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 47206 T41 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 47207 T41 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 47211 T41 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 47212 T41 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 47216 T41 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 47217 T41 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 47222 T41 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 47223 T41 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 47227 T41 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 47228 T41 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 48603 T41 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/control-001/cores/collection1
   [junit4]   2> 48610 T41 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 48614 T41 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:41353
   [junit4]   2> 48615 T41 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/control-001/cores, hostPort=32680, hostContext=/vpe/ei}
   [junit4]   2> 48616 T41 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 48617 T41 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/control-001/'
   [junit4]   2> 48686 T41 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/control-001/solr.xml
   [junit4]   2> 48772 T41 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/control-001/cores
   [junit4]   2> 48773 T41 oasc.CoreContainer.<init> New CoreContainer 136382826
   [junit4]   2> 48774 T41 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/control-001/]
   [junit4]   2> 48774 T41 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/control-001/lib
   [junit4]   2> 48776 T41 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/control-001/lib).
   [junit4]   2> 48791 T41 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 48792 T41 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 48792 T41 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 48793 T41 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 48793 T41 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 48794 T41 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 48794 T41 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 48795 T41 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 48796 T41 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 48796 T41 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 48797 T41 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 48798 T41 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 48798 T41 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 48799 T41 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 48800 T41 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 48800 T41 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:24732/solr
   [junit4]   2> 48885 T41 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 48886 T41 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 48887 T41 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 48890 T64 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@30028ac8 name:ZooKeeperConnection Watcher:127.0.0.1:24732 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 48890 T41 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 48892 T41 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 48930 T41 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 48933 T67 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@530d8dcc name:ZooKeeperConnection Watcher:127.0.0.1:24732/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 48934 T41 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 48954 T41 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 48972 T41 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 48984 T41 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 48989 T41 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 48993 T41 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 49024 T41 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 49029 T41 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 49032 T41 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 49035 T41 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:32680_vpe%2Fei
   [junit4]   2> 49043 T41 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:32680_vpe%2Fei
   [junit4]   2> 49048 T41 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 49052 T41 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 49056 T41 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 49081 T41 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:32680_vpe%2Fei
   [junit4]   2> 49081 T41 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 49086 T41 oasc.Overseer.start Overseer (id=93556307864387587-127.0.0.1:32680_vpe%2Fei-n_0000000000) starting
   [junit4]   2> 49112 T41 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 49166 T41 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 49202 T69 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 49202 T41 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 49250 T68 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 49263 T41 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/control-001/cores
   [junit4]   2> 49290 T41 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/control-001/cores/collection1, collection=control_collection, shard=, dataDir=data/, config=solrconfig.xml, transient=false, coreNodeName=, name=collection1, loadOnStartup=true, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/control-001/cores/collection1/}
   [junit4]   2> 49291 T41 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/control-001/cores/collection1/
   [junit4]   2> 49292 T41 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 49295 T71 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 49295 T71 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 49320 T67 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 49320 T71 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 49334 T68 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:32680_vpe%2Fei",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32680/vpe/ei"} current state version: 0
   [junit4]   2> 49409 T68 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:32680_vpe%2Fei",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32680/vpe/ei"}
   [junit4]   2> 49418 T68 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 49492 T68 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 49523 T67 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 50323 T71 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 50323 T71 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 50325 T71 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 50325 T71 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 50327 T71 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 50328 T71 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/control-001/cores/collection1/'
   [junit4]   2> 50376 T71 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 50398 T71 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 50434 T71 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 50482 T71 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 50543 T71 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 50546 T71 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 50581 T71 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 51110 T71 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 51133 T71 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 51137 T71 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 51171 T71 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 51177 T71 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 51184 T71 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 51185 T71 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 51186 T71 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 51186 T71 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 51188 T71 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 51189 T71 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 51189 T71 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 51190 T71 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 51190 T71 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/control-001/cores/collection1/data/
   [junit4]   2> 51191 T71 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@72abdc66
   [junit4]   2> 51310 T71 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/control-001/cores/collection1/data
   [junit4]   2> 51311 T71 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/control-001/cores/collection1/data/index/
   [junit4]   2> 51311 T71 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 51312 T71 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/control-001/cores/collection1/data/index
   [junit4]   2> 51312 T71 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=2, maxMergeAtOnceExplicit=6, maxMergedSegmentMB=44.4248046875, floorSegmentMB=1.7158203125, forceMergeDeletesPctAllowed=1.4873689293662262, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7656066649276347
   [junit4]   2> 51314 T71 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@1540d1c3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@528b641b),segFN=segments_1,generation=1}
   [junit4]   2> 51314 T71 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 51328 T71 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 51399 T71 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 51399 T71 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 51400 T71 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 51400 T71 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 51401 T71 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 51402 T71 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 51403 T71 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 51403 T71 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 51424 T71 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 51435 T71 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 51438 T71 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 51441 T71 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 51461 T71 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/segments,/schema,standard,/admin/luke,/update/json/docs,/admin/properties,/update/csv,/admin/threads,/get,/update,/admin/logging,/admin/file,/admin/plugins,/replication,/config,/admin/ping,/admin/system,/update/json,/admin/mbeans
   [junit4]   2> 51469 T71 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 51471 T71 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 51471 T71 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 51473 T71 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 51473 T71 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 51475 T71 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@7aa30390
   [junit4]   2> 51476 T71 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@1540d1c3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@528b641b),segFN=segments_1,generation=1}
   [junit4]   2> 51476 T71 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 51477 T71 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@6cc69c04[collection1] main
   [junit4]   2> 51478 T71 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 51480 T71 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 51480 T71 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 51492 T71 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 51492 T71 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 51493 T71 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 51494 T71 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 51495 T71 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 51495 T71 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 51496 T71 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 51498 T72 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6cc69c04[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 51503 T71 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 51504 T75 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:32680/vpe/ei collection:control_collection shard:shard1
   [junit4]   2> 51505 T41 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 51505 T41 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 51540 T75 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 51548 T75 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 51550 T67 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 51551 T75 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 51552 T75 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 51552 T68 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 51553 T41 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 51554 T41 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 51556 T79 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1a3f9f3b name:ZooKeeperConnection Watcher:127.0.0.1:24732/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 51556 T41 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 51557 T41 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 51558 T41 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C5 name=collection1 org.apache.solr.core.SolrCore@ea2d3b7 url=http://127.0.0.1:32680/vpe/ei/collection1 node=127.0.0.1:32680_vpe%2Fei C5_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, node_name=127.0.0.1:32680_vpe%2Fei, base_url=http://127.0.0.1:32680/vpe/ei}
   [junit4]   2> 51560 T75 C:control_collection S:shard1 C5 P32680 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:32680/vpe/ei/collection1/
   [junit4]   2> 51560 T75 C:control_collection S:shard1 C5 P32680 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 51561 T75 C:control_collection S:shard1 C5 P32680 oasc.SyncStrategy.syncToMe http://127.0.0.1:32680/vpe/ei/collection1/ has no replicas
   [junit4]   2> 51561 T75 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:32680/vpe/ei/collection1/ shard1
   [junit4]   2> 51564 T79 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 51564 T67 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 51567 T41 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 51567 T41 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 51568 T41 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 51569 T41 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 51570 T82 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1bcfbe92 name:ZooKeeperConnection Watcher:127.0.0.1:24732/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 51571 T41 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 51571 T41 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 51573 T75 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 51574 T67 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 51577 T68 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 51578 T68 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 51584 T68 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32680/vpe/ei",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 51586 T68 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 51589 T67 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 51590 T79 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 51598 T83 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 51600 T79 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 51600 T67 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 51634 T75 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:32680/vpe/ei/collection1/ and leader is http://127.0.0.1:32680/vpe/ei/collection1/
   [junit4]   2> 51634 T75 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:32680/vpe/ei
   [junit4]   2> 51635 T75 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 51636 T75 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 51636 T75 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 51639 T67 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 51641 T68 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:32680_vpe%2Fei",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32680/vpe/ei"} current state version: 4
   [junit4]   2> 51643 T68 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:32680_vpe%2Fei",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:32680/vpe/ei"}
   [junit4]   2> 51750 T67 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 51750 T79 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 52821 T41 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/shard-1-001/cores/collection1
   [junit4]   2> 52824 T41 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/shard-1-001
   [junit4]   2> 52825 T41 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 52828 T41 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:15401
   [junit4]   2> 52828 T41 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/shard-1-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/tempDir-001/jetty1, hostPort=20580, hostContext=/vpe/ei}
   [junit4]   2> 52829 T41 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 52829 T41 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/shard-1-001/'
   [junit4]   2> 52875 T41 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/shard-1-001/solr.xml
   [junit4]   2> 52936 T41 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/shard-1-001/cores
   [junit4]   2> 52936 T41 oasc.CoreContainer.<init> New CoreContainer 908212679
   [junit4]   2> 52936 T41 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/shard-1-001/]
   [junit4]   2> 52947 T41 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/shard-1-001/lib
   [junit4]   2> 52948 T41 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/shard-1-001/lib).
   [junit4]   2> 52958 T41 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 52959 T41 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 52959 T41 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 52960 T41 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 52960 T41 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 52960 T41 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 52961 T41 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 52961 T41 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 52961 T41 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 52962 T41 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 52962 T41 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 52963 T41 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 52963 T41 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 52963 T41 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 52964 T41 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 52964 T41 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:24732/solr
   [junit4]   2> 52964 T41 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 52965 T41 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 52966 T41 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 52968 T95 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@44288c6d name:ZooKeeperConnection Watcher:127.0.0.1:24732 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 52969 T41 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 52970 T41 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 52973 T41 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 52976 T98 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@e2b52b7 name:ZooKeeperConnection Watcher:127.0.0.1:24732/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 52976 T41 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 52986 T41 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 53999 T41 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:20580_vpe%2Fei
   [junit4]   2> 54001 T41 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:20580_vpe%2Fei
   [junit4]   2> 54007 T41 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 54020 T41 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/shard-1-001/cores
   [junit4]   2> 54023 T41 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/shard-1-001/cores/collection1/, coreNodeName=, collection=collection1, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/shard-1-001/cores/collection1, dataDir=data/, transient=false, schema=schema.xml, loadOnStartup=true, shard=, name=collection1}
   [junit4]   2> 54023 T41 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/shard-1-001/cores/collection1/
   [junit4]   2> 54024 T41 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 54029 T99 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 54030 T99 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 54034 T67 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 54036 T99 C:collection1 c:collection1 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 54036 T68 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:20580_vpe%2Fei",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20580/vpe/ei"} current state version: 5
   [junit4]   2> 54036 T99 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 54037 T68 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:20580_vpe%2Fei",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20580/vpe/ei"}
   [junit4]   2> 54038 T99 C:collection1 c:collection1 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 54038 T68 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 54039 T68 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 54042 T68 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 54052 T99 C:collection1 c:collection1 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 54052 T99 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 54052 T99 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 54053 T99 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 54054 T99 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 54054 T99 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 54056 T99 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 54056 T99 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/shard-1-001/cores/collection1/'
   [junit4]   2> 54076 T99 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 54081 T99 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 54099 T99 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 54131 T99 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 54203 T99 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 54206 T99 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 54232 T99 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 54777 T99 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 54803 T99 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 54806 T99 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 54844 T99 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 54851 T99 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 54857 T99 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 54860 T99 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 54860 T99 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 54861 T99 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 54863 T99 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 54863 T99 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 54864 T99 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 54864 T99 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 54865 T99 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 54865 T99 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@72abdc66
   [junit4]   2> 54867 T99 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/shard-1-001/cores/collection1/data
   [junit4]   2> 54868 T99 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 54868 T99 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 54869 T99 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 7FF19CB2C80AC917-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 54870 T99 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=2, maxMergeAtOnceExplicit=6, maxMergedSegmentMB=44.4248046875, floorSegmentMB=1.7158203125, forceMergeDeletesPctAllowed=1.4873689293662262, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7656066649276347
   [junit4]   2> 54871 T99 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@a824b57 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@70204224),segFN=segments_1,generation=1}
   [junit4]   2> 54872 T99 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 54889 T99 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 54889 T99 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 54890 T99 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 54890 T99 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 54891 T99 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 54891 T99 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 54892 T99 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 54893 T99 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 54893 T99 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 54923 T99 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 54927 T99 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 54932 T99 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 54936 T99 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 54976 T99 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/segments,/schema,standard,/admin/luke,/update/json/docs,/admin/properties,/update/csv,/admin/threads,/get,/update,/admin/logging,/admin/file,/admin/plugins,/replication,/config,/admin/ping,/admin/system,/update/json,/admin/mbeans
   [junit4]   2> 54979 T99 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 54981 T99 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 54981 T99 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 54983 T99 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 54984 T99 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 54986 T99 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@2aa0d8c1
   [junit4]   2> 54987 T99 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@a824b57 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@70204224),segFN=segments_1,generation=1}
   [junit4]   2> 54987 T99 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 54988 T99 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@59793b54[collection1] main
   [junit4]   2> 54989 T99 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 54991 T99 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 54991 T99 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 54992 T99 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 54992 T99 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 54993 T99 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 54994 T99 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 54995 T99 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 54995 T99 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 54995 T99 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 54999 T100 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@59793b54[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 55000 T99 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 55001 T103 C:collection1 S:shard3 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:20580/vpe/ei collection:collection1 shard:shard3
   [junit4]   2> 55002 T41 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 55003 T41 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 55003 T103 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 55012 T103 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard3
   [junit4]   2> 55015 T67 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 55016 T103 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 55016 T103 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C6 name=collection1 org.apache.solr.core.SolrCore@649dd18c url=http://127.0.0.1:20580/vpe/ei/collection1 node=127.0.0.1:20580_vpe%2Fei C6_STATE=coll:collection1 core:collection1 props:{core=collection1, state=down, node_name=127.0.0.1:20580_vpe%2Fei, base_url=http://127.0.0.1:20580/vpe/ei}
   [junit4]   2> 55017 T103 C:collection1 S:shard3 C6 P20580 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:20580/vpe/ei/collection1/
   [junit4]   2> 55017 T68 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 55017 T103 C:collection1 S:shard3 C6 P20580 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 55018 T103 C:collection1 S:shard3 C6 P20580 oasc.SyncStrategy.syncToMe http://127.0.0.1:20580/vpe/ei/collection1/ has no replicas
   [junit4]   2> 55019 T103 C:collection1 S:shard3 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:20580/vpe/ei/collection1/ shard3
   [junit4]   2> 55019 T103 C:collection1 S:shard3 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
   [junit4]   2> 55020 T68 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 55021 T98 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 55023 T98 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 55028 T68 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20580/vpe/ei",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 55030 T68 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 55031 T98 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 55032 T98 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 55035 T67 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 55078 T103 C:collection1 S:shard3 oasc.ZkController.register We are http://127.0.0.1:20580/vpe/ei/collection1/ and leader is http://127.0.0.1:20580/vpe/ei/collection1/
   [junit4]   2> 55079 T103 C:collection1 S:shard3 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:20580/vpe/ei
   [junit4]   2> 55079 T103 C:collection1 S:shard3 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 55079 T103 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 55079 T103 C:collection1 S:shard3 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 55082 T67 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 55084 T68 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:20580_vpe%2Fei",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20580/vpe/ei"} current state version: 5
   [junit4]   2> 55085 T68 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:20580_vpe%2Fei",
   [junit4]   2> 	 

[...truncated too long message...]

derInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:56864/c8n_1x3_commits_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([7FF19CB2C80AC917:F7A5A36866F6A4EF]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:960)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:935)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 2458178 T13464 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 7FF19CB2C80AC917-001
   [junit4]   2> 78585 T13463 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=is, timezone=America/Indiana/Vincennes
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=126102328,total=462422016
   [junit4]   2> NOTE: All tests run in this JVM: [TestBlobHandler, TestQueryTypes, TestConfigReload, ShardRoutingTest, BadIndexSchemaTest, MergeStrategyTest, OutOfBoxZkACLAndCredentialsProvidersTest, TestIntervalFaceting, TestFieldCollectionResource, TestDistributedGrouping, TermVectorComponentDistributedTest, TestRandomMergePolicy, TestWordDelimiterFilterFactory, SampleTest, TestCSVLoader, CoreAdminRequestStatusTest, TestQuerySenderListener, TestIndexSearcher, TestComplexPhraseQParserPlugin, RecoveryAfterSoftCommitTest, TestJoin, SpatialHeatmapFacetsTest, TestDefaultSimilarityFactory, SpellingQueryConverterTest, TestValueSourceCache, UnloadDistributedZkTest, TermsComponentTest, TestDefaultSearchFieldResource, TestStressUserVersions, DateMathParserTest, DistributedFacetPivotWhiteBoxTest, TestJmxIntegration, InfoHandlerTest, TestSchemaManager, ScriptEngineTest, OutputWriterTest, TestSolr4Spatial2, TestWriterPerf, TestSolrQueryParserResource, TestGroupingSearch, TestFieldTypeResource, TestRangeQuery, TestTrackingShardHandlerFactory, DistributedIntervalFacetingTest, TestBulkSchemaConcurrent, TestObjectReleaseTracker, SSLMigrationTest, FileUtilsTest, TestRebalanceLeaders, LeaderElectionTest, TestInitQParser, DocValuesMultiTest, TestCursorMarkWithoutUniqueKey, XsltUpdateRequestHandlerTest, TestCodecSupport, TestBulkSchemaAPI, CSVRequestHandlerTest, TestManagedSchema, OverseerTest, BasicZkTest, FullSolrCloudDistribCmdsTest, TestFaceting, TestHashPartitioner, DistributedSpellCheckComponentTest, TestReload, DistributedTermsComponentTest, TestSolr4Spatial, SolrCoreTest, SolrCmdDistributorTest, TestSort, TestFiltering, TestFunctionQuery, BasicFunctionalityTest, TestLazyCores, DirectUpdateHandlerTest, SoftAutoCommitTest, CurrencyFieldXmlFileTest, SimplePostToolTest, TestCoreDiscovery, AnalysisAfterCoreReloadTest, SignatureUpdateProcessorFactoryTest, SpellCheckCollatorTest, SuggesterFSTTest, CoreAdminHandlerTest, DocValuesTest, SuggesterTSTTest, SuggesterTest, TestStressLucene, SpatialFilterTest, SuggesterWFSTTest, PolyFieldTest, NoCacheHeaderTest, SchemaVersionSpecificBehaviorTest, SolrCoreCheckLockOnStartupTest, TestPseudoReturnFields, TestUpdate, TestAtomicUpdateErrorCases, DefaultValueUpdateProcessorTest, SortByFunctionTest, DistanceFunctionTest, TestSolrDeletionPolicy1, SolrInfoMBeanTest, DebugComponentTest, CacheHeaderTest, TestSurroundQueryParser, TestQueryUtils, DocumentAnalysisRequestHandlerTest, TestOmitPositions, RequestHandlersTest, TestIndexingPerformance, RequiredFieldsTest, FieldAnalysisRequestHandlerTest, TestArbitraryIndexDir, FastVectorHighlighterTest, IndexSchemaRuntimeFieldTest, SolrPluginUtilsTest, TestCollationField, UpdateRequestProcessorFactoryTest, JSONWriterTest, TestCSVResponseWriter, QueryParsingTest, TestAnalyzedSuggestions, JsonLoaderTest, BinaryUpdateRequestHandlerTest, TestComponentsName, SearchHandlerTest, TestBinaryResponseWriter, HighlighterConfigTest, UpdateParamsTest, AlternateDirectoryTest, TestSolrIndexConfig, TestQuerySenderNoQuery, ResponseLogComponentTest, SolrIndexConfigTest, BadComponentTest, MultiTermTest, TestSearchPerf, NumericFieldsTest, TestElisionMultitermQuery, TestConfig, TestPostingsSolrHighlighter, DirectSolrConnectionTest, NotRequiredUniqueKeyTest, TestLuceneMatchVersion, TestPhraseSuggestions, SpellPossibilityIteratorTest, SynonymTokenizerTest, TestXIncludeConfig, EchoParamsTest, TestLMDirichletSimilarityFactory, TestBM25SimilarityFactory, TestJmxMonitoredMap, TimeZoneUtilsTest, ResourceLoaderTest, OpenExchangeRatesOrgProviderTest, URLClassifyProcessorTest, PluginInfoTest, TestSystemIdResolver, PrimUtilsTest, TestSuggestSpellingConverter, RAMDirectoryFactoryTest, ClusterStateTest, TestUtils, ZkNodePropsTest, SystemInfoHandlerTest, UUIDFieldTest, CircularListTest, DistributedMLTComponentTest, TestRTGBase, TestDistributedMissingSort, TestHighlightDedupGrouping, TestSimpleTrackingShardHandler, TestEmbeddedSolrServerConstructors, TestJettySolrRunner, AliasIntegrationTest, CustomCollectionTest, DistribJoinFromCollectionTest, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J3 in 79.62s, 1 test, 1 error <<< FAILURES!

[...truncated 325 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 3 errors, 58 ignored (9 assumptions)

Total time: 162 minutes 6 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 864408841 bytes
Compression is 0.0%
Took 2 min 39 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 799 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/799/

5 tests failed.
REGRESSION:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:13238/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:13238/collection1
	at __randomizedtesting.SeedInfo.seed([D557C075CD234EE9:5D03FFAF63DF2311]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:224)
	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:166)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:671)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test(FullSolrCloudDistribCmdsTest.java:151)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketException: Broken pipe
	at java.net.SocketOutputStream.socketWrite0(Native Method)
	at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:113)
	at java.net.SocketOutputStream.write(SocketOutputStream.java:159)
	at org.apache.http.impl.io.AbstractSessionOutputBuffer.flushBuffer(AbstractSessionOutputBuffer.java:159)
	at org.apache.http.impl.io.AbstractSessionOutputBuffer.writeLine(AbstractSessionOutputBuffer.java:258)
	at org.apache.http.impl.io.HttpRequestWriter.writeHeadLine(HttpRequestWriter.java:59)
	at org.apache.http.impl.io.HttpRequestWriter.writeHeadLine(HttpRequestWriter.java:46)
	at org.apache.http.impl.io.AbstractMessageWriter.write(AbstractMessageWriter.java:109)
	at org.apache.http.impl.AbstractHttpClientConnection.sendRequestHeader(AbstractHttpClientConnection.java:254)
	at org.apache.http.impl.conn.DefaultClientConnection.sendRequestHeader(DefaultClientConnection.java:270)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.sendRequestHeader(ManagedClientConnectionImpl.java:209)
	at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:203)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:121)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 50 more


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.MultiThreadedOCPTest

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.MultiThreadedOCPTest:     1) Thread[id=2613, name=OverseerThreadFactory-358-thread-5, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerCollectionProcessor.waitForCoreNodeName(OverseerCollectionProcessor.java:1851)         at org.apache.solr.cloud.OverseerCollectionProcessor.splitShard(OverseerCollectionProcessor.java:1737)         at org.apache.solr.cloud.OverseerCollectionProcessor.processMessage(OverseerCollectionProcessor.java:626)         at org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2895)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.MultiThreadedOCPTest: 
   1) Thread[id=2613, name=OverseerThreadFactory-358-thread-5, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerCollectionProcessor.waitForCoreNodeName(OverseerCollectionProcessor.java:1851)
        at org.apache.solr.cloud.OverseerCollectionProcessor.splitShard(OverseerCollectionProcessor.java:1737)
        at org.apache.solr.cloud.OverseerCollectionProcessor.processMessage(OverseerCollectionProcessor.java:626)
        at org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2895)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([D557C075CD234EE9]:0)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:46102/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:46102/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([D557C075CD234EE9:5D03FFAF63DF2311]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([D557C075CD234EE9:5D03FFAF63DF2311]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([D557C075CD234EE9:5D03FFAF63DF2311]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 10107 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/init-core-data-001
   [junit4]   2> 3157 T13 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 3243 T13 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 3256 T14 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 3657 T13 oasc.ZkTestServer.run start zk server on port:10521
   [junit4]   2> 3811 T13 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 4083 T13 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 4288 T21 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@25370a40 name:ZooKeeperConnection Watcher:127.0.0.1:10521 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 4289 T13 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 4305 T13 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 4322 T13 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 4480 T13 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 4482 T13 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 4484 T24 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e67f94e name:ZooKeeperConnection Watcher:127.0.0.1:10521/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 4485 T13 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 4494 T13 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 4531 T13 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 4536 T13 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 4540 T13 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 4556 T13 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 4561 T13 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 4627 T13 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 4634 T13 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 4636 T13 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 4642 T13 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 4643 T13 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 4648 T13 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 4649 T13 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 4654 T13 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 4655 T13 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 4660 T13 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 4661 T13 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 4666 T13 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 4667 T13 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 4672 T13 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 4673 T13 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 4677 T13 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 4679 T13 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 4683 T13 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 4685 T13 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 4689 T13 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 4690 T13 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 7569 T13 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/control-001/cores/collection1
   [junit4]   2> 8280 T13 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 8616 T13 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:56393
   [junit4]   2> 8617 T13 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=60060, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/control-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/tempDir-001/control/data}
   [junit4]   2> 9053 T13 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 9196 T13 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/control-001/'
   [junit4]   2> 10394 T13 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/control-001/solr.xml
   [junit4]   2> 10668 T13 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/control-001/cores
   [junit4]   2> 11164 T13 oasc.CoreContainer.<init> New CoreContainer 815188297
   [junit4]   2> 11165 T13 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/control-001/]
   [junit4]   2> 11166 T13 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/control-001/lib
   [junit4]   2> 11167 T13 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/control-001/lib).
   [junit4]   2> 11471 T13 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 11472 T13 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 11484 T13 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 11485 T13 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 11486 T13 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 11486 T13 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 11487 T13 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 11487 T13 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 11488 T13 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 11489 T13 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 11489 T13 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 12118 T13 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 12136 T13 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 12143 T13 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 12155 T13 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 12156 T13 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:10521/solr
   [junit4]   2> 12227 T13 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 12228 T13 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 12229 T13 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 12234 T37 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7f641270 name:ZooKeeperConnection Watcher:127.0.0.1:10521 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 12235 T13 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 12237 T13 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 12265 T13 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 12268 T40 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@204a755a name:ZooKeeperConnection Watcher:127.0.0.1:10521/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 12268 T13 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 12289 T13 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 12307 T13 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 12320 T13 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 12326 T13 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 12332 T13 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 12373 T13 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 12378 T13 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 12382 T13 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 12386 T13 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:60060_
   [junit4]   2> 12398 T13 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:60060_
   [junit4]   2> 12404 T13 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 12408 T13 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 12412 T13 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 12444 T13 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:60060_
   [junit4]   2> 12445 T13 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 12450 T13 oasc.Overseer.start Overseer (id=93550709067546627-127.0.0.1:60060_-n_0000000000) starting
   [junit4]   2> 12482 T13 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 12583 T13 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 13226 T42 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 13226 T13 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 13409 T41 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 13753 T13 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/control-001/cores
   [junit4]   2> 14215 T13 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, transient=false, collection=control_collection, name=collection1, schema=schema.xml, shard=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/control-001/cores/collection1, config=solrconfig.xml, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/control-001/cores/collection1/, loadOnStartup=true}
   [junit4]   2> 14223 T13 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/control-001/cores/collection1/
   [junit4]   2> 14224 T13 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 14240 T44 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 14241 T44 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 14264 T40 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 14264 T44 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 14273 T41 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:60060",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:60060_",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 0
   [junit4]   2> 14341 T41 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:60060",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:60060_",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 14351 T41 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 14417 T41 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 14441 T40 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 15266 T44 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 15267 T44 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 15268 T44 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 15269 T44 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 15271 T44 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 15281 T44 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/control-001/cores/collection1/'
   [junit4]   2> 15353 T44 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 15386 T44 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 15487 T44 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 15686 T44 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 16008 T44 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 16053 T44 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 16266 T44 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 18557 T44 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 18684 T44 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 18694 T44 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 20029 T44 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 20062 T44 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 20069 T44 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 20077 T44 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 20078 T44 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 20078 T44 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 20080 T44 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 20081 T44 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 20101 T44 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 20102 T44 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 20135 T44 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/control-001/cores/collection1/data/
   [junit4]   2> 20152 T44 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5b381d19
   [junit4]   2> 20581 T44 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/control-001/cores/collection1/data
   [junit4]   2> 20582 T44 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/control-001/cores/collection1/data/index/
   [junit4]   2> 20583 T44 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 20583 T44 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/control-001/cores/collection1/data/index
   [junit4]   2> 20591 T44 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@55e6703
   [junit4]   2> 20985 T44 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2a8eb8b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@14ddc),segFN=segments_1,generation=1}
   [junit4]   2> 20993 T44 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 23415 T44 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 23732 T44 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 23773 T44 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 23774 T44 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 23774 T44 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 23775 T44 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 24004 T44 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 24005 T44 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 24006 T44 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 24377 T44 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 24425 T44 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 24428 T44 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 24432 T44 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 24557 T44 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,/admin/logging,/replication,/admin/plugins,/admin/segments,/update/json/docs,/update/json,/admin/threads,/config,/admin/mbeans,/admin/file,/update,standard,/update/csv,/admin/luke,/admin/ping,/get,/schema,/admin/properties
   [junit4]   2> 24566 T44 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 24687 T44 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 24688 T44 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 24736 T44 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 24737 T44 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 24739 T44 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=28, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 24758 T44 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@2a8eb8b8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@14ddc),segFN=segments_1,generation=1}
   [junit4]   2> 24759 T44 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 24930 T44 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@68d6c743[collection1] main
   [junit4]   2> 24973 T44 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 24976 T44 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 24977 T44 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 25002 T44 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 25003 T44 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 25202 T44 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 25204 T44 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 25205 T44 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 25206 T44 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 25503 T44 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 25516 T45 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@68d6c743[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 25526 T44 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 25536 T48 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:60060 collection:control_collection shard:shard1
   [junit4]   2> 25536 T13 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 25537 T13 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 25586 T48 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 25596 T48 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 25600 T40 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 25600 T48 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 25601 T48 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 25602 T41 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 25618 T40 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> ASYNC  NEW_CORE C0 name=collection1 org.apache.solr.core.SolrCore@40d2370d url=http://127.0.0.1:60060/collection1 node=127.0.0.1:60060_ C0_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=http://127.0.0.1:60060, state=down, node_name=127.0.0.1:60060_}
   [junit4]   2> 25632 T48 C:control_collection S:shard1 C0 P60060 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:60060/collection1/
   [junit4]   2> 25641 T48 C:control_collection S:shard1 C0 P60060 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 25642 T48 C:control_collection S:shard1 C0 P60060 oasc.SyncStrategy.syncToMe http://127.0.0.1:60060/collection1/ has no replicas
   [junit4]   2> 25643 T48 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:60060/collection1/ shard1
   [junit4]   2> 25661 T48 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 25670 T40 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 25673 T41 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:60060",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 25687 T13 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 25689 T13 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 25691 T52 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@12aa77d3 name:ZooKeeperConnection Watcher:127.0.0.1:10521/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 25692 T13 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 25693 T13 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 25694 T13 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 25703 T13 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 25779 T52 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 25779 T40 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 25825 T48 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:60060/collection1/ and leader is http://127.0.0.1:60060/collection1/
   [junit4]   2> 25835 T48 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:60060
   [junit4]   2> 25835 T48 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 25836 T48 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 25836 T48 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 25839 T40 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 25842 T41 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:60060",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:60060_",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 3
   [junit4]   2> 25844 T41 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:60060",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "node_name":"127.0.0.1:60060_",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 25951 T52 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 25951 T40 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 27575 T13 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/shard-1-001/cores/collection1
   [junit4]   2> 27580 T13 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/shard-1-001
   [junit4]   2> 27581 T13 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 27584 T13 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:53429
   [junit4]   2> 27584 T13 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/tempDir-001/jetty1, hostPort=19482, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/shard-1-001/cores, hostContext=/, solrconfig=solrconfig.xml}
   [junit4]   2> 27585 T13 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 27586 T13 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/shard-1-001/'
   [junit4]   2> 27643 T13 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/shard-1-001/solr.xml
   [junit4]   2> 27722 T13 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/shard-1-001/cores
   [junit4]   2> 27723 T13 oasc.CoreContainer.<init> New CoreContainer 2009141278
   [junit4]   2> 27724 T13 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/shard-1-001/]
   [junit4]   2> 27724 T13 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/shard-1-001/lib
   [junit4]   2> 27725 T13 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/shard-1-001/lib).
   [junit4]   2> 27740 T13 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 27740 T13 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 27741 T13 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 27741 T13 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 27742 T13 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 27742 T13 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 27743 T13 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 27743 T13 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 27744 T13 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 27744 T13 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 27745 T13 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 27746 T13 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 27746 T13 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 27747 T13 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 27747 T13 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 27748 T13 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:10521/solr
   [junit4]   2> 27748 T13 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 27749 T13 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 27750 T13 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 27753 T64 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6148874d name:ZooKeeperConnection Watcher:127.0.0.1:10521 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 27753 T13 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 27754 T13 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 27757 T13 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 27759 T67 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@10b7415b name:ZooKeeperConnection Watcher:127.0.0.1:10521/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 27760 T13 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 27768 T13 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 28776 T13 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:19482_
   [junit4]   2> 28778 T13 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:19482_
   [junit4]   2> 28783 T13 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 28794 T13 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/shard-1-001/cores
   [junit4]   2> 28797 T13 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/shard-1-001/cores/collection1/, dataDir=data/, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/shard-1-001/cores/collection1, schema=schema.xml, name=collection1, transient=false, coreNodeName=, loadOnStartup=true, shard=, collection=collection1}
   [junit4]   2> 28798 T13 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/shard-1-001/cores/collection1/
   [junit4]   2> 28798 T13 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 28800 T68 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 28800 T68 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 28802 T40 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 28802 T68 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 28803 T41 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19482",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:19482_",
   [junit4]   2> 	  "collection":"collection1"} current state version: 4
   [junit4]   2> 28804 T41 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:19482",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "node_name":"127.0.0.1:19482_",
   [junit4]   2> 	  "collection":"collection1"}
   [junit4]   2> 28805 T41 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 28805 T41 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 28808 T40 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 28808 T67 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 28808 T52 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 29803 T68 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 29804 T68 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 29805 T68 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 29806 T68 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 29807 T68 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 29807 T68 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/shard-1-001/cores/collection1/'
   [junit4]   2> 29825 T68 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 29828 T68 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 29851 T68 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 29879 T68 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 29939 T68 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 29941 T68 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 30027 T68 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 30484 T68 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 30511 T68 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 30515 T68 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 30548 T68 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 30555 T68 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 30561 T68 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 30563 T68 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 30563 T68 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 30564 T68 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 30565 T68 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 30566 T68 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 30566 T68 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 30567 T68 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 30567 T68 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 30568 T68 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5b381d19
   [junit4]   2> 30570 T68 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/shard-1-001/cores/collection1/data
   [junit4]   2> 30570 T68 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 30570 T68 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 30571 T68 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest D557C075CD234EE9-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 30572 T68 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@729bce74
   [junit4]   2> 30573 T68 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@157eee46 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@404d40b0),segFN=segments_1,generation=1}
   [junit4]   2> 30573 T68 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 30588 T68 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 30589 T68 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 30590 T68 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 30590 T68 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 30591 T68 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 30591 T68 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 30592 T68 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 30593 T68 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 30593 T68 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 30617 T68 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 30621 T68 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 30626 T68 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 30630 T68 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 30667 T68 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,/admin/logging,/replication,/admin/plugins,/admin/segments,/update/json/docs,/update/json,/admin/threads,/config,/admin/mbeans,/admin/file,/update,standard,/update/csv,/admin/luke,/admin/ping,/get,/schema,/admin/properties
   [junit4]   2> 30670 T68 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 30672 T68 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 30673 T68 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 30674 T68 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 30675 T68 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 30677 T68 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=28, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0]
   [junit4]   2> 30678 T68 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@157eee46 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@404d40b0),segFN=segments_1,generation=1}
   [junit4]   2> 30679 T68 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 30679 T68 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@1f3027a0[collection1] main
   [junit4]   2> 30680 T68 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 30682 T68 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 30682 T68 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 30683 T68 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 30683 T68 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 30684 T68 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 30685 T68 C:collection1 c:collection1 oasr.Manag

[...truncated too long message...]

 2> 2055941 T5512 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:10254/_jx/p
   [junit4]   2> 2055942 T5512 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:10244/_jx/p
   [junit4]   2> 2055942 T5512 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 10 connections to: http://127.0.0.1:10250/_jx/p
   [junit4]   2> 2055942 T5512 C:control_collection S:shard1 c:collection1 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 2055942 T5512 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:10258/_jx/p
   [junit4]   2> 2055943 T5512 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:10254/_jx/p
   [junit4]   2> 2055943 T5512 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:10244/_jx/p
   [junit4]   2> 2055943 T5512 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:10250/_jx/p
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=D557C075CD234EE9 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=ko -Dtests.timezone=SystemV/AST4 -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   77.2s J0 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([D557C075CD234EE9:5D03FFAF63DF2311]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 2055951 T5512 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest D557C075CD234EE9-001
   [junit4]   2> 77217 T5511 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=FastCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST, chunkSize=9, maxDocsPerChunk=133, blockSize=440), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST, chunkSize=9, blockSize=440)), sim=DefaultSimilarity, locale=ko, timezone=SystemV/AST4
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=171005728,total=399507456
   [junit4]   2> NOTE: All tests run in this JVM: [TestFastLRUCache, LeaderInitiatedRecoveryOnCommitTest, TestCloudManagedSchemaConcurrent, IndexBasedSpellCheckerTest, TestPartialUpdateDeduplication, TestMissingGroups, RequestHandlersTest, RecoveryZkTest, MultiThreadedOCPTest, QueryParsingTest, TestSolrJ, ShardRoutingCustomTest, OverseerRolesTest, TestSolrDeletionPolicy2, TestIBSimilarityFactory, DistributedQueryComponentOptimizationTest, SignatureUpdateProcessorFactoryTest, BJQParserTest, TestRandomFaceting, SimpleCollectionCreateDeleteTest, TestDFRSimilarityFactory, DeleteReplicaTest, RequiredFieldsTest, TestDynamicFieldResource, TestCloudManagedSchema, TestUpdate, DocExpirationUpdateProcessorFactoryTest, TestFreeTextSuggestions, SolrCmdDistributorTest, TestImplicitCoreProperties, TimeZoneUtilsTest, SpellCheckCollatorTest, DebugComponentTest, TestFileDictionaryLookup, UUIDFieldTest, TestMiniSolrCloudClusterSSL, TestCollationFieldDocValues, ZkNodePropsTest, TestStandardQParsers, RegexBoostProcessorTest, AsyncMigrateRouteKeyTest, FastVectorHighlighterTest, ParsingFieldUpdateProcessorsTest, SimpleFacetsTest, OverriddenZkACLAndCredentialsProvidersTest, TestPivotHelperCode, DistributedSpellCheckComponentTest, HttpPartitionTest, HighlighterConfigTest, TestLeaderElectionZkExpiry, DistributedFacetPivotLargeTest, TestConfigSets, HdfsDirectoryTest, PolyFieldTest, TestSolrCoreProperties, RequestLoggingTest, TestSchemaResource, ReplicationFactorTest]
   [junit4] Completed on J0 in 77.75s, 1 test, 1 error <<< FAILURES!

[...truncated 755 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 478 suites, 1902 tests, 1 suite-level error, 4 errors, 58 ignored (9 assumptions)

Total time: 169 minutes 41 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 858562164 bytes
Compression is 0.0%
Took 2 min 36 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 798 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/798/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:51483/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51483/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([806EDD83CDD8EC98:83AE25963248160]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([806EDD83CDD8EC98:83AE25963248160]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([806EDD83CDD8EC98:83AE25963248160]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 10143 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/init-core-data-001
   [junit4]   2> 192015 T559 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /_/r
   [junit4]   2> 192020 T559 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 192021 T560 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 192120 T559 oasc.ZkTestServer.run start zk server on port:62477
   [junit4]   2> 192121 T559 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 192122 T559 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 192125 T567 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7f97bad6 name:ZooKeeperConnection Watcher:127.0.0.1:62477 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 192125 T559 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 192125 T559 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 192125 T559 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 192128 T559 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 192128 T559 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 192130 T570 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@71526773 name:ZooKeeperConnection Watcher:127.0.0.1:62477/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 192130 T559 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 192130 T559 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 192130 T559 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 192132 T559 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 192134 T559 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 192135 T559 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 192137 T559 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 192137 T559 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 192140 T559 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 192140 T559 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 192143 T559 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 192143 T559 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 192145 T559 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 192145 T559 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 192147 T559 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 192147 T559 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 192149 T559 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 192149 T559 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 192151 T559 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 192151 T559 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 192153 T559 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 192153 T559 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 192155 T559 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 192156 T559 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 192158 T559 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 192159 T559 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 192161 T559 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 192161 T559 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 193253 T559 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/control-001/cores/collection1
   [junit4]   2> 193257 T559 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 193260 T559 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:62488
   [junit4]   2> 193261 T559 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/tempDir-001/control/data, hostContext=/_/r, hostPort=62487}
   [junit4]   2> 193261 T559 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 193261 T559 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/control-001/'
   [junit4]   2> 193298 T559 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/control-001/solr.xml
   [junit4]   2> 193338 T559 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/control-001/cores
   [junit4]   2> 193339 T559 oasc.CoreContainer.<init> New CoreContainer 2056787055
   [junit4]   2> 193339 T559 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/control-001/]
   [junit4]   2> 193339 T559 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/control-001/lib
   [junit4]   2> 193340 T559 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/control-001/lib).
   [junit4]   2> 193349 T559 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 193349 T559 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 193349 T559 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 193349 T559 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 193350 T559 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 193350 T559 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 193350 T559 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 193350 T559 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 193350 T559 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 193351 T559 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 193351 T559 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 193351 T559 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 193351 T559 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 193352 T559 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 193352 T559 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 193352 T559 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:62477/solr
   [junit4]   2> 193352 T559 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 193353 T559 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 193353 T559 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 193355 T582 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@56f45373 name:ZooKeeperConnection Watcher:127.0.0.1:62477 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 193356 T559 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 193356 T559 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 193358 T559 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 193359 T585 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@75491ada name:ZooKeeperConnection Watcher:127.0.0.1:62477/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 193360 T559 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 193361 T559 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 193364 T559 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 193366 T559 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 193368 T559 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 193371 T559 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 193373 T559 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 193375 T559 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 193377 T559 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 193378 T559 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62487__%2Fr
   [junit4]   2> 193379 T559 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:62487__%2Fr
   [junit4]   2> 193381 T559 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 193382 T559 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 193384 T559 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 193386 T559 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:62487__%2Fr
   [junit4]   2> 193386 T559 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 193388 T559 oasc.Overseer.start Overseer (id=93544991241666563-127.0.0.1:62487__%2Fr-n_0000000000) starting
   [junit4]   2> 193390 T559 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 193398 T559 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 193398 T587 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 193398 T559 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 193399 T586 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 193402 T559 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/control-001/cores
   [junit4]   2> 193404 T559 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=control_collection, schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/control-001/cores/collection1/, name=collection1, transient=false, coreNodeName=, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/control-001/cores/collection1, shard=, loadOnStartup=true, dataDir=data/}
   [junit4]   2> 193405 T559 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/control-001/cores/collection1/
   [junit4]   2> 193405 T559 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 193406 T589 C:control_collection c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 193406 T589 C:control_collection c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 193407 T589 C:control_collection c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 193407 T585 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 193408 T586 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62487/_/r",
   [junit4]   2> 	  "node_name":"127.0.0.1:62487__%2Fr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 193409 T586 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62487/_/r",
   [junit4]   2> 	  "node_name":"127.0.0.1:62487__%2Fr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 193409 T586 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 193409 T586 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 193410 T585 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 194407 T589 C:control_collection c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 194408 T589 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 194408 T589 C:control_collection c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 194409 T589 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 194409 T589 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 194409 T589 C:control_collection c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/control-001/cores/collection1/'
   [junit4]   2> 194422 T589 C:control_collection c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 194424 T589 C:control_collection c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 194432 T589 C:control_collection c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 194447 T589 C:control_collection c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 194481 T589 C:control_collection c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 194482 T589 C:control_collection c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 194490 T589 C:control_collection c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 195059 T589 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 195071 T589 C:control_collection c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 195074 T589 C:control_collection c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 195086 T589 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 195090 T589 C:control_collection c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 195093 T589 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 195094 T589 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 195094 T589 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 195094 T589 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 195095 T589 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 195095 T589 C:control_collection c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 195095 T589 C:control_collection c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 195095 T589 C:control_collection c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 195095 T589 C:control_collection c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/control-001/cores/collection1/data/
   [junit4]   2> 195096 T589 C:control_collection c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@381cc7bf
   [junit4]   2> 195096 T589 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/control-001/cores/collection1/data
   [junit4]   2> 195097 T589 C:control_collection c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/control-001/cores/collection1/data/index/
   [junit4]   2> 195097 T589 C:control_collection c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 195097 T589 C:control_collection c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/control-001/cores/collection1/data/index
   [junit4]   2> 195097 T589 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=8, maxMergeAtOnceExplicit=8, maxMergedSegmentMB=92.94140625, floorSegmentMB=0.6015625, forceMergeDeletesPctAllowed=12.31110864872323, segmentsPerTier=3.0, maxCFSSegmentSizeMB=1.796875, noCFSRatio=1.0
   [junit4]   2> 195098 T589 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@32863a08 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@a4bfd22),segFN=segments_1,generation=1}
   [junit4]   2> 195098 T589 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 195103 T589 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 195103 T589 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 195104 T589 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 195104 T589 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 195104 T589 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 195104 T589 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 195104 T589 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 195105 T589 C:control_collection c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 195105 T589 C:control_collection c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 195119 T589 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 195121 T589 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 195123 T589 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 195124 T589 C:control_collection c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 195128 T589 C:control_collection c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /replication,/update/json/docs,/admin/properties,/schema,/admin/mbeans,/update/csv,/admin/logging,/admin/system,/update/json,/admin/luke,/update,/config,/admin/threads,standard,/admin/plugins,/admin/file,/admin/segments,/admin/ping,/get
   [junit4]   2> 195129 T589 C:control_collection c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 195131 T589 C:control_collection c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 195131 T589 C:control_collection c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 195132 T589 C:control_collection c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 195132 T589 C:control_collection c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 195132 T589 C:control_collection c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=9, maxMergeAtOnceExplicit=3, maxMergedSegmentMB=13.2822265625, floorSegmentMB=1.81640625, forceMergeDeletesPctAllowed=18.778154855961255, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 195133 T589 C:control_collection c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@32863a08 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@a4bfd22),segFN=segments_1,generation=1}
   [junit4]   2> 195133 T589 C:control_collection c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 195133 T589 C:control_collection c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@4e67e01a[collection1] main
   [junit4]   2> 195134 T589 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 195135 T589 C:control_collection c:collection1 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 195135 T589 C:control_collection c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 195135 T589 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 195135 T589 C:control_collection c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 195135 T589 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 195136 T589 C:control_collection c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 195136 T589 C:control_collection c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 195136 T589 C:control_collection c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 195136 T589 C:control_collection c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 195137 T590 C:control_collection c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@4e67e01a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 195137 T589 C:control_collection c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 195138 T593 C:control_collection S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:62487/_/r collection:control_collection shard:shard1
   [junit4]   2> 195139 T559 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 195139 T559 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 195139 T593 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 195142 T559 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 195142 T559 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 195143 T597 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2dc14888 name:ZooKeeperConnection Watcher:127.0.0.1:62477/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 195144 T559 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 195144 T593 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 195144 T559 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 195144 T559 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 195146 T585 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 195146 T593 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 195146 T593 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C115 name=collection1 org.apache.solr.core.SolrCore@1553558d url=http://127.0.0.1:62487/_/r/collection1 node=127.0.0.1:62487__%2Fr C115_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:62487/_/r, node_name=127.0.0.1:62487__%2Fr, core=collection1}
   [junit4]   2> 195147 T593 C:control_collection S:shard1 C115 P62487 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:62487/_/r/collection1/
   [junit4]   2> 195147 T586 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 195147 T593 C:control_collection S:shard1 C115 P62487 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 195147 T593 C:control_collection S:shard1 C115 P62487 oasc.SyncStrategy.syncToMe http://127.0.0.1:62487/_/r/collection1/ has no replicas
   [junit4]   2> 195147 T593 C:control_collection S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:62487/_/r/collection1/ shard1
   [junit4]   2> 195147 T593 C:control_collection S:shard1 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 195148 T559 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 195148 T585 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 195148 T597 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 195152 T585 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 195152 T586 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62487/_/r",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 195255 T597 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 195255 T585 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 195304 T593 C:control_collection S:shard1 oasc.ZkController.register We are http://127.0.0.1:62487/_/r/collection1/ and leader is http://127.0.0.1:62487/_/r/collection1/
   [junit4]   2> 195305 T593 C:control_collection S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:62487/_/r
   [junit4]   2> 195305 T593 C:control_collection S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 195305 T593 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 195305 T593 C:control_collection S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 195306 T585 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 195307 T586 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62487/_/r",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:62487__%2Fr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state"} current state version: 3
   [junit4]   2> 195308 T586 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62487/_/r",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:62487__%2Fr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 195411 T585 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 195411 T597 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 196348 T559 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/shard-1-001/cores/collection1
   [junit4]   2> 196350 T559 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/shard-1-001
   [junit4]   2> 196352 T559 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 196353 T559 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:62495
   [junit4]   2> 196354 T559 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/shard-1-001/cores, hostContext=/_/r, hostPort=62494}
   [junit4]   2> 196354 T559 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@60da5686
   [junit4]   2> 196354 T559 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/shard-1-001/'
   [junit4]   2> 196388 T559 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/shard-1-001/solr.xml
   [junit4]   2> 196452 T559 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/shard-1-001/cores
   [junit4]   2> 196453 T559 oasc.CoreContainer.<init> New CoreContainer 1849509443
   [junit4]   2> 196453 T559 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/shard-1-001/]
   [junit4]   2> 196453 T559 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/shard-1-001/lib
   [junit4]   2> 196454 T559 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/shard-1-001/lib).
   [junit4]   2> 196467 T559 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 196467 T559 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 196467 T559 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 196468 T559 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 196468 T559 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 196468 T559 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 196469 T559 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 196469 T559 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 196469 T559 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 196470 T559 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 196470 T559 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 196471 T559 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 196471 T559 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 196471 T559 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 196472 T559 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 196472 T559 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:62477/solr
   [junit4]   2> 196472 T559 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 196473 T559 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 196473 T559 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 196475 T609 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7173782d name:ZooKeeperConnection Watcher:127.0.0.1:62477 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 196476 T559 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 196476 T559 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 196479 T559 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 196481 T612 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3fb3985e name:ZooKeeperConnection Watcher:127.0.0.1:62477/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 196481 T559 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 196488 T559 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 197493 T559 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:62494__%2Fr
   [junit4]   2> 197494 T559 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:62494__%2Fr
   [junit4]   2> 197497 T559 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 197500 T559 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/shard-1-001/cores
   [junit4]   2> 197502 T559 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {collection=collection1, transient=false, shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/shard-1-001/cores/collection1/, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/shard-1-001/cores/collection1, config=solrconfig.xml, coreNodeName=, name=collection1, dataDir=data/, loadOnStartup=true}
   [junit4]   2> 197502 T559 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/shard-1-001/cores/collection1/
   [junit4]   2> 197502 T559 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 197503 T613 C:collection1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 197503 T613 C:collection1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 197504 T585 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 197504 T613 C:collection1 c:collection1 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 197505 T586 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62494/_/r",
   [junit4]   2> 	  "node_name":"127.0.0.1:62494__%2Fr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 197505 T586 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62494/_/r",
   [junit4]   2> 	  "node_name":"127.0.0.1:62494__%2Fr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 197506 T586 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 197506 T586 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 197507 T612 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 197507 T597 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 197507 T585 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 198504 T613 C:collection1 c:collection1 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 198505 T613 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 198505 T613 C:collection1 c:collection1 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 198506 T613 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 198507 T613 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 198507 T613 C:collection1 c:collection1 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/shard-1-001/cores/collection1/'
   [junit4]   2> 198519 T613 C:collection1 c:collection1 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 198521 T613 C:collection1 c:collection1 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 198529 T613 C:collection1 c:collection1 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 198545 T613 C:collection1 c:collection1 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 198578 T613 C:collection1 c:collection1 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 198580 T613 C:collection1 c:collection1 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 198587 T613 C:collection1 c:collection1 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 198858 T613 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 198870 T613 C:collection1 c:collection1 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 198872 T613 C:collection1 c:collection1 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 198886 T613 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 198889 T613 C:collection1 c:collection1 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 198893 T613 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 198894 T613 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 198894 T613 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 198894 T613 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 198895 T613 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 198895 T613 C:collection1 c:collection1 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 198895 T613 C:collection1 c:collection1 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 198895 T613 C:collection1 c:collection1 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 198895 T613 C:collection1 c:collection1 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 198896 T613 C:collection1 c:collection1 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@381cc7bf
   [junit4]   2> 198896 T613 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/shard-1-001/cores/collection1/data
   [junit4]   2> 198897 T613 C:collection1 c:collection1 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 198897 T613 C:collection1 c:collection1 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 198897 T613 C:collection1 c:collection1 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 198897 T613 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=8, maxMergeAtOnceExplicit=8, maxMergedSegmentMB=92.94140625, floorSegmentMB=0.6015625, forceMergeDeletesPctAllowed=12.31110864872323, segmentsPerTier=3.0, maxCFSSegmentSizeMB=1.796875, noCFSRatio=1.0
   [junit4]   2> 198898 T613 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5fddc74a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5946fc71),segFN=segments_1,generation=1}
   [junit4]   2> 198898 T613 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 198904 T613 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 198904 T613 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 198904 T613 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 198905 T613 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 198905 T613 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 198905 T613 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 198905 T613 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 198905 T613 C:collection1 c:collection1 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 198906 T613 C:collection1 c:collection1 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 198922 T613 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 198923 T613 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 198925 T613 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 198927 T613 C:collection1 c:collection1 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 198932 T613 C:collection1 c:collection1 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /replication,/update/json/docs,/admin/properties,/schema,/admin/mbeans,/update/csv,/admin/logging,/admin/system,/update/json,/admin/luke,/update,/config,/admin/threads,standard,/admin/plugins,/admin/file,/admin/segments,/admin/ping,/get
   [junit4]   2> 198934 T613 C:collection1 c:collection1 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 198935 T613 C:collection1 c:collection1 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 198936 T613 C:collection1 c:collection1 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 198937 T613 C:collection1 c:collection1 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 198937 T613 C:collection1 c:collection1 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 198938 T613 C:collection1 c:collection1 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=9, maxMergeAtOnceExplicit=3, maxMergedSegmentMB=13.2822265625, floorSegmentMB=1.81640625, forceMergeDeletesPctAllowed=18.778154855961255, segmentsPerTier=28.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 198938 T613 C:collection1 c:collection1 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5fddc74a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5946fc71),segFN=segments_1,generation=1}
   [junit4]   2> 198938 T613 C:collection1 c:collection1 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 198939 T613 C:collection1 c:collection1 oass.SolrIndexSearcher.<init> Opening Searcher@211a4999[collection1] main
   [junit4]   2> 198939 T613 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 198940 T613 C:collection1 c:collection1 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 198940 T613 C:collection1 c:collection1 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 198941 T613 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 198941 T613 C:collection1 c:collection1 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 198941 T613 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 198942 T613 C:collection1 c:collection1 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 198942 T613 C:collection1 c:collection1 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 198942 T613 C:collection1 c:collection1 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 198942 T613 C:collection1 c:collection1 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 198943 T614 C:collection1 c:collection1 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@211a4999[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 198944 T613 C:collection1 c:collection1 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 198944 T617 C:collection1 S:shard1 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:62494/_/r collection:collection1 shard:shard1
   [junit4]   2> 198945 T559 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 198945 T559 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 198945 T617 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 198951 T617 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 198952 T585 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 198953 T617 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 198953 T617 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C116 name=collection1 org.apache.solr.core.SolrCore@7e8904 url=http://127.0.0.1:62494/_/r/collection1 node=127.0.0.1:62494__%2Fr C116_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:62494/_/r, node_name=127.0.0.1:62494__%2Fr, core=collection1}
   [junit4]   2> 198953 T617 C:collection1 S:shard1 C116 P62494 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:62494/_/r/collection1/
   [junit4]   2> 198953 T617 C:collection1 S:shard1 C116 P62494 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 198953 T586 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 198953 T617 C:collection1 S:shard1 C116 P62494 oasc.SyncStrategy.syncToMe http://127.0.0.1:62494/_/r/collection1/ has no replicas
   [junit4]   2> 198953 T617 C:collection1 S:shard1 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:62494/_/r/collection1/ shard1
   [junit4]   2> 198954 T617 C:collection1 S:shard1 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 198958 T585 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 198959 T586 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62494/_/r",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 199063 T585 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 199063 T597 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 199063 T612 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 199111 T617 C:collection1 S:shard1 oasc.ZkController.register We are http://127.0.0.1:62494/_/r/collection1/ and leader is http://127.0.0.1:62494/_/r/collection1/
   [junit4]   2> 199112 T617 C:collection1 S:shard1 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:62494/_/r
   [junit4]   2> 199112 T617 C:collection1 S:shard1 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 199112 T617 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 199113 T617 C:collection1 S:shard1 c:collection1 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 199115 T585 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 199116 T586 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62494/_/r",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:62494__%2Fr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state"} current state version: 6
   [junit4]   2> 199117 T586 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:62494/_/r",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "node_name":"127.0.0.1:62494__%2Fr",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 199222 T612 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 199222 T585 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 199222 T597 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 200681 T559 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/shard-2-001/cores/collection1
   [junit4]   2> 200684 T559 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/shard-2-001
   [junit4]   2> 200685 T559 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 200688 T559 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:62505
   [junit4]   2> 200688 T559 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=62504, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 806EDD83CDD8EC98-001/tempDir-001/jetty2, solrc

[...truncated too long message...]

erver.send4LetterWord connecting to 127.0.0.1:51408 51408
   [junit4]   2> 2461787 T4316 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/clusterstate.json
   [junit4]   2> 		6	/solr/aliases.json
   [junit4]   2> 		5	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/c8n_1x3_commits/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		6	/solr/live_nodes
   [junit4]   2> 		5	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 2461787 T4315 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51418/
   [junit4]   2> 2461787 T4315 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51457/
   [junit4]   2> 2461788 T4315 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 4 connections to: http://127.0.0.1:51483/
   [junit4]   2> 2461788 T4315 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51469/
   [junit4]   2> 2461788 T4315 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 13 connections to: http://127.0.0.1:51439/
   [junit4]   2> 2461789 T4315 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51418/
   [junit4]   2> 2461789 T4315 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51457/
   [junit4]   2> 2461790 T4315 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51483/
   [junit4]   2> 2461790 T4315 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51469/
   [junit4]   2> 2461790 T4315 C:control_collection S:shard1 c:collection1 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:51439/
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderInitiatedRecoveryOnCommitTest -Dtests.method=test -Dtests.seed=806EDD83CDD8EC98 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=ru_RU -Dtests.timezone=America/Montserrat -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   74.9s J1 | LeaderInitiatedRecoveryOnCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51483/c8n_1x3_commits_shard1_replica2
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([806EDD83CDD8EC98:83AE25963248160]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:570)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:233)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:225)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
   [junit4]    > 	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:462)
   [junit4]    > 	... 48 more
   [junit4]   2> 2461807 T4315 C:control_collection S:shard1 c:collection1 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 806EDD83CDD8EC98-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=ru_RU, timezone=America/Montserrat
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=153493608,total=499122176
   [junit4]   2> NOTE: All tests run in this JVM: [SOLR749Test, TestCryptoKeys, TestConfigOverlay, SuggesterFSTTest, TestSchemaNameResource, TestFunctionQuery, TestBlendedInfixSuggestions, DistributedTermsComponentTest, CursorMarkTest, TestNRTOpen, TestSolrDeletionPolicy1, TestSweetSpotSimilarityFactory, TestInitQParser, SchemaVersionSpecificBehaviorTest, HighlighterTest, ShardSplitTest, ExitableDirectoryReaderTest, TestLRUCache, TestChildDocTransformer, CoreAdminRequestStatusTest, OpenCloseCoreStressTest, SortByFunctionTest, TestZkChroot, TestSortingResponseWriter, TestElisionMultitermQuery, DocumentAnalysisRequestHandlerTest, CircularListTest, TestQuerySenderNoQuery, TermVectorComponentTest, TestCollectionAPI, TestSolr4Spatial2, AlternateDirectoryTest, DistributedExpandComponentTest, TestSolrConfigHandlerCloud, DistributedQueryComponentCustomSortTest, TestStressVersions, QueryParsingTest, ScriptEngineTest, TestFastWriter, TestPostingsSolrHighlighter, SolrInfoMBeanTest, EnumFieldTest, AtomicUpdatesTest, AliasIntegrationTest, TestJettySolrRunner, SolrTestCaseJ4Test, TestLMJelinekMercerSimilarityFactory, AutoCommitTest, RemoteQueryErrorTest, WordBreakSolrSpellCheckerTest, TestDocBasedVersionConstraints, SimplePostToolTest, CollectionsAPIAsyncDistributedZkTest, TestSolr4Spatial, NotRequiredUniqueKeyTest, TestMaxScoreQueryParser, ShardRoutingTest, TestReplicationHandler, LeaderInitiatedRecoveryOnCommitTest]
   [junit4] Completed on J1 in 74.91s, 1 test, 1 error <<< FAILURES!

[...truncated 336 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 477 suites, 1899 tests, 3 errors, 58 ignored (9 assumptions)

Total time: 163 minutes 40 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 824016145 bytes
Compression is 0.0%
Took 2 min 34 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 797 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/797/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:11914/pt/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11914/pt/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([9A810A1455DE44C8:12D535CEFB222930]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38885/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38885/collection1
	at __randomizedtesting.SeedInfo.seed([9A810A1455DE44C8:12D535CEFB222930]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38885/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18764/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18764/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([9A810A1455DE44C8:12D535CEFB222930]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18764/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 53 more




Build Log:
[...truncated 10167 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/init-core-data-001
   [junit4]   2> 38222 T67 [] [] [] []  oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 38281 T67 [] [] [] []  oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 38292 T68 [] [] [] []  oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 38695 T67 [] [] [] []  oasc.ZkTestServer.run start zk server on port:17323
   [junit4]   2> 38946 T67 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 39291 T67 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 39559 T75 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6d1ee1cb name:ZooKeeperConnection Watcher:127.0.0.1:17323 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 39560 T67 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 39570 T67 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 39585 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 39778 T67 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 39780 T67 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 39782 T78 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@27509e09 name:ZooKeeperConnection Watcher:127.0.0.1:17323/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 39783 T67 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 39784 T67 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 39805 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 39811 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 39815 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 39819 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 39824 T67 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 39826 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 39833 T67 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 39834 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 39840 T67 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 39841 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 39847 T67 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 39848 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 39853 T67 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 39854 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 39859 T67 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 39860 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 39865 T67 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 39866 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 39871 T67 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 39872 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 39877 T67 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 39878 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 39883 T67 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 39884 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 39888 T67 [] [] [] []  oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 39889 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 41764 T67 [] [] [] []  oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/control-001/cores/collection1
   [junit4]   2> 41769 T67 [] [] [] []  oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 41772 T67 [] [] [] []  oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:26400
   [junit4]   2> 41773 T67 [] [] [] []  oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/control-001/cores, hostPort=34485, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/tempDir-001/control/data}
   [junit4]   2> 41773 T67 [] [] [] []  oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 41774 T67 [] [] [] []  oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/control-001/'
   [junit4]   2> 41824 T67 [] [] [] []  oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/control-001/solr.xml
   [junit4]   2> 41888 T67 [] [] [] []  oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/control-001/cores
   [junit4]   2> 41888 T67 [] [] [] []  oasc.CoreContainer.<init> New CoreContainer 1256947391
   [junit4]   2> 41889 T67 [] [] [] []  oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/control-001/]
   [junit4]   2> 41889 T67 [] [] [] []  oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/control-001/lib
   [junit4]   2> 41890 T67 [] [] [] []  oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/control-001/lib).
   [junit4]   2> 41902 T67 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 41902 T67 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 41903 T67 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 41903 T67 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 41903 T67 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 41904 T67 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 41904 T67 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 41905 T67 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 41905 T67 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 41905 T67 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 41906 T67 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 41906 T67 [] [] [] []  oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 41907 T67 [] [] [] []  oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 41907 T67 [] [] [] []  oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 41908 T67 [] [] [] []  oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 41908 T67 [] [] [] []  oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:17323/solr
   [junit4]   2> 41959 T67 [] [] [] []  oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 41959 T67 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 41960 T67 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 41963 T90 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e17c516 name:ZooKeeperConnection Watcher:127.0.0.1:17323 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 41963 T67 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 41964 T67 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 41992 T67 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 41994 T93 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@18b64ed5 name:ZooKeeperConnection Watcher:127.0.0.1:17323/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 41994 T67 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 42016 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 42034 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 42044 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 42048 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 42051 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 42083 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 42086 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 42088 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 42090 T67 [] [] [] []  oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:34485_
   [junit4]   2> 42099 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:34485_
   [junit4]   2> 42104 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 42106 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 42109 T67 [] [] [] []  oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 42133 T67 [] [] [] []  oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:34485_
   [junit4]   2> 42134 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 42137 T67 [] [] [] []  oasc.Overseer.start Overseer (id=93539606741647363-127.0.0.1:34485_-n_0000000000) starting
   [junit4]   2> 42160 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 42199 T67 [] [] [] []  oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 42737 T67 [] [] [] []  oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 42737 T95 [] [] [] []  oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 42757 T94 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 42763 T67 [] [] [] []  oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/control-001/cores
   [junit4]   2> 42778 T67 [] [] [] []  oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {transient=false, name=collection1, coreNodeName=, collection=control_collection, config=solrconfig.xml, dataDir=data/, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/control-001/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/control-001/cores/collection1/, schema=schema.xml, shard=}
   [junit4]   2> 42778 T67 [] [] [] []  oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/control-001/cores/collection1/
   [junit4]   2> 42779 T67 [] [] [] []  oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 42780 T97 [control_collection] [] [] [collection1]  oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 42781 T97 [control_collection] [] [] [collection1]  oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 42795 T93 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 42796 T97 [control_collection] [] [] [collection1]  oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 42804 T94 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:34485_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:34485"} current state version: 0
   [junit4]   2> 42899 T94 [] [] [] []  oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:34485_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:34485"}
   [junit4]   2> 42911 T94 [] [] [] []  oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 42984 T94 [] [] [] []  oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 43010 T93 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 43797 T97 [control_collection] [] [] [collection1]  oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 43798 T97 [control_collection] [] [] [collection1]  oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 43800 T97 [control_collection] [] [] [collection1]  oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 43800 T97 [control_collection] [] [] [collection1]  oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 43803 T97 [control_collection] [] [] [collection1]  oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 43804 T97 [control_collection] [] [] [collection1]  oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/control-001/cores/collection1/'
   [junit4]   2> 43912 T97 [control_collection] [] [] [collection1]  oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 43957 T97 [control_collection] [] [] [collection1]  oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 44034 T97 [control_collection] [] [] [collection1]  oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 44079 T97 [control_collection] [] [] [collection1]  oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 44175 T97 [control_collection] [] [] [collection1]  oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 44178 T97 [control_collection] [] [] [collection1]  oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 44237 T97 [control_collection] [] [] [collection1]  oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 45825 T97 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 45923 T97 [control_collection] [] [] [collection1]  oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 45928 T97 [control_collection] [] [] [collection1]  oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 46036 T97 [control_collection] [] [] [collection1]  oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 46045 T97 [control_collection] [] [] [collection1]  oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 46053 T97 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 46068 T97 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 46068 T97 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 46070 T97 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 46071 T97 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 46072 T97 [control_collection] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 46110 T97 [control_collection] [] [] [collection1]  oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 46111 T97 [control_collection] [] [] [collection1]  oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 46111 T97 [control_collection] [] [] [collection1]  oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/control-001/cores/collection1/data/
   [junit4]   2> 46112 T97 [control_collection] [] [] [collection1]  oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@31657222
   [junit4]   2> 46356 T97 [control_collection] [] [] [collection1]  oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/control-001/cores/collection1/data
   [junit4]   2> 46357 T97 [control_collection] [] [] [collection1]  oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/control-001/cores/collection1/data/index/
   [junit4]   2> 46357 T97 [control_collection] [] [] [collection1]  oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 46358 T97 [control_collection] [] [] [collection1]  oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/control-001/cores/collection1/data/index
   [junit4]   2> 46359 T97 [control_collection] [] [] [collection1]  oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=12, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.24691946960350455]
   [junit4]   2> 46376 T97 [control_collection] [] [] [collection1]  oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@7d25da76 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4029b92a; maxCacheMB=0.66015625 maxMergeSizeMB=0.83203125)),segFN=segments_1,generation=1}
   [junit4]   2> 46377 T97 [control_collection] [] [] [collection1]  oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 46391 T97 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 46491 T97 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 46492 T97 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 46492 T97 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 46492 T97 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 46493 T97 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 46494 T97 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 46494 T97 [control_collection] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 46495 T97 [control_collection] [] [] [collection1]  oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 46526 T97 [control_collection] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 46538 T97 [control_collection] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 46541 T97 [control_collection] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 46544 T97 [control_collection] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 46559 T97 [control_collection] [] [] [collection1]  oasc.RequestHandlers.initHandlersFromConfig Registered paths: standard,/admin/system,/update/csv,/admin/luke,/admin/properties,/schema,/replication,/get,/update/json/docs,/config,/update/json,/admin/plugins,/update,/admin/segments,/admin/threads,/admin/logging,/admin/ping,/admin/file,/admin/mbeans
   [junit4]   2> 46569 T97 [control_collection] [] [] [collection1]  oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 46571 T97 [control_collection] [] [] [collection1]  oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 46572 T97 [control_collection] [] [] [collection1]  oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 46574 T97 [control_collection] [] [] [collection1]  oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 46574 T97 [control_collection] [] [] [collection1]  oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 46576 T97 [control_collection] [] [] [collection1]  oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=874432736, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 46577 T97 [control_collection] [] [] [collection1]  oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@7d25da76 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4029b92a; maxCacheMB=0.66015625 maxMergeSizeMB=0.83203125)),segFN=segments_1,generation=1}
   [junit4]   2> 46577 T97 [control_collection] [] [] [collection1]  oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 46578 T97 [control_collection] [] [] [collection1]  oass.SolrIndexSearcher.<init> Opening Searcher@29f38115[collection1] main
   [junit4]   2> 46578 T97 [control_collection] [] [] [collection1]  oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 46580 T97 [control_collection] [] [] [collection1]  oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 46581 T97 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 46596 T97 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 46596 T97 [control_collection] [] [] [collection1]  oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 46597 T97 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 46598 T97 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 46598 T97 [control_collection] [] [] [collection1]  oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 46598 T97 [control_collection] [] [] [collection1]  oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 46599 T97 [control_collection] [] [] [collection1]  oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 46602 T98 [control_collection] [] [] [collection1]  oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@29f38115[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 46610 T97 [control_collection] [] [] [collection1]  oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 46612 T101 [control_collection] [shard1] [] []  oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:34485 collection:control_collection shard:shard1
   [junit4]   2> 46612 T67 [] [] [] []  oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 46613 T67 [] [] [] []  oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 46659 T101 [control_collection] [shard1] [] []  oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 46669 T101 [control_collection] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 46671 T93 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 46672 T101 [control_collection] [shard1] [] []  oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 46673 T101 [control_collection] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 46673 T94 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C4 name=collection1 org.apache.solr.core.SolrCore@7cf33f9e url=http://127.0.0.1:34485/collection1 node=127.0.0.1:34485_ C4_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:34485_, state=down, core=collection1, base_url=http://127.0.0.1:34485}
   [junit4]   2> 46686 T101 [control_collection] [shard1] [] []  C4 P34485 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:34485/collection1/
   [junit4]   2> 46687 T101 [control_collection] [shard1] [] []  C4 P34485 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 46687 T101 [control_collection] [shard1] [] []  C4 P34485 oasc.SyncStrategy.syncToMe http://127.0.0.1:34485/collection1/ has no replicas
   [junit4]   2> 46687 T101 [control_collection] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:34485/collection1/ shard1
   [junit4]   2> 46691 T93 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 46707 T101 [control_collection] [shard1] [] []  oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 46714 T93 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 46716 T94 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:34485",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 46777 T67 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 46778 T67 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 46781 T105 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1f15e9a8 name:ZooKeeperConnection Watcher:127.0.0.1:17323/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 46781 T67 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 46782 T67 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 46783 T67 [] [] [] []  oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 46793 T67 [] [] [] []  oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 46794 T67 [] [] [] []  oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 46795 T67 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 46796 T67 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 46798 T108 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1042ce9b name:ZooKeeperConnection Watcher:127.0.0.1:17323/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 46799 T67 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 46800 T67 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 46803 T93 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 46806 T94 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 46806 T94 [] [] [] []  oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 46808 T105 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 46808 T93 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 46817 T101 [control_collection] [shard1] [] []  oasc.ZkController.register We are http://127.0.0.1:34485/collection1/ and leader is http://127.0.0.1:34485/collection1/
   [junit4]   2> 46818 T101 [control_collection] [shard1] [] []  oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:34485
   [junit4]   2> 46818 T101 [control_collection] [shard1] [] []  oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 46819 T101 [control_collection] [shard1] [] [collection1]  oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 46819 T101 [control_collection] [shard1] [] [collection1]  oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 46821 T93 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 46824 T94 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:34485_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:34485"} current state version: 3
   [junit4]   2> 46825 T94 [] [] [] []  oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:34485_",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:34485"}
   [junit4]   2> 46827 T94 [] [] [] []  oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 46830 T93 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 46830 T105 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 46940 T105 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 46940 T93 [] [] [] []  oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 48120 T67 [] [] [] []  oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/shard-1-001/cores/collection1
   [junit4]   2> 48123 T67 [] [] [] []  oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/shard-1-001
   [junit4]   2> 48124 T67 [] [] [] []  oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 48126 T67 [] [] [] []  oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:55740
   [junit4]   2> 48126 T67 [] [] [] []  oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=38885, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/shard-1-001/cores, hostContext=/, solrconfig=solrconfig.xml}
   [junit4]   2> 48127 T67 [] [] [] []  oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 48127 T67 [] [] [] []  oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/shard-1-001/'
   [junit4]   2> 48162 T67 [] [] [] []  oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/shard-1-001/solr.xml
   [junit4]   2> 48207 T67 [] [] [] []  oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/shard-1-001/cores
   [junit4]   2> 48208 T67 [] [] [] []  oasc.CoreContainer.<init> New CoreContainer 1265460015
   [junit4]   2> 48208 T67 [] [] [] []  oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/shard-1-001/]
   [junit4]   2> 48209 T67 [] [] [] []  oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/shard-1-001/lib
   [junit4]   2> 48210 T67 [] [] [] []  oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/shard-1-001/lib).
   [junit4]   2> 48220 T67 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 48221 T67 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 48221 T67 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 48222 T67 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 48222 T67 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 48222 T67 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 48223 T67 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 48223 T67 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 48223 T67 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 48224 T67 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 48224 T67 [] [] [] []  oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 48225 T67 [] [] [] []  oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 48225 T67 [] [] [] []  oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 48226 T67 [] [] [] []  oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 48226 T67 [] [] [] []  oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 48227 T67 [] [] [] []  oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:17323/solr
   [junit4]   2> 48227 T67 [] [] [] []  oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 48227 T67 [] [] [] []  oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 48228 T67 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 48231 T120 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@436444e5 name:ZooKeeperConnection Watcher:127.0.0.1:17323 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 48232 T67 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 48233 T67 [] [] [] []  oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 48235 T67 [] [] [] []  oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 48237 T123 [] [] [] []  oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6fd92134 name:ZooKeeperConnection Watcher:127.0.0.1:17323/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 48237 T67 [] [] [] []  oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 48245 T67 [] [] [] []  oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 49255 T67 [] [] [] []  oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:38885_
   [junit4]   2> 49256 T67 [] [] [] []  oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:38885_
   [junit4]   2> 49261 T67 [] [] [] []  oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 49272 T67 [] [] [] []  oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/shard-1-001/cores
   [junit4]   2> 49274 T67 [] [] [] []  oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/shard-1-001/cores/collection1, dataDir=data/, coreNodeName=, schema=schema.xml, config=solrconfig.xml, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/shard-1-001/cores/collection1/, shard=, collection=collection1, transient=false}
   [junit4]   2> 49275 T67 [] [] [] []  oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/shard-1-001/cores/collection1/
   [junit4]   2> 49275 T67 [] [] [] []  oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 49279 T124 [collection1] [] [] [collection1]  oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 49279 T124 [collection1] [] [] [collection1]  oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 49281 T93 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 49282 T124 [collection1] [] [] [collection1]  oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 49283 T124 [collection1] [] [] [collection1]  oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 49283 T94 [] [] [] []  oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:38885_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38885"} current state version: 5
   [junit4]   2> 49283 T124 [collection1] [] [] [collection1]  oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 49284 T94 [] [] [] []  oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:38885_",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:38885"}
   [junit4]   2> 49285 T94 [] [] [] []  oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 49285 T94 [] [] [] []  oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 49289 T94 [] [] [] []  oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 49294 T124 [collection1] [] [] [collection1]  oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 49294 T124 [collection1] [] [] [collection1]  oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 49295 T124 [collection1] [] [] [collection1]  oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 49295 T124 [collection1] [] [] [collection1]  oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 49296 T124 [collection1] [] [] [collection1]  oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 49296 T124 [collection1] [] [] [collection1]  oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 49298 T124 [collection1] [] [] [collection1]  oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 49299 T124 [collection1] [] [] [collection1]  oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/shard-1-001/cores/collection1/'
   [junit4]   2> 49313 T124 [collection1] [] [] [collection1]  oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 49317 T124 [collection1] [] [] [collection1]  oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 49334 T124 [collection1] [] [] [collection1]  oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 49359 T124 [collection1] [] [] [collection1]  oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 49414 T124 [collection1] [] [] [collection1]  oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 49417 T124 [collection1] [] [] [collection1]  oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 49466 T124 [collection1] [] [] [collection1]  oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 50087 T124 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 50124 T124 [collection1] [] [] [collection1]  oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 50129 T124 [collection1] [] [] [collection1]  oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 50172 T124 [collection1] [] [] [collection1]  oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 50181 T124 [collection1] [] [] [collection1]  oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 50189 T124 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 50191 T124 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 50192 T124 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 50192 T124 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 50194 T124 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 50194 T124 [collection1] [] [] [collection1]  oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 50195 T124 [collection1] [] [] [collection1]  oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 50196 T124 [collection1] [] [] [collection1]  oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 50196 T124 [collection1] [] [] [collection1]  oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 50197 T124 [collection1] [] [] [collection1]  oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@31657222
   [junit4]   2> 50199 T124 [collection1] [] [] [collection1]  oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/shard-1-001/cores/collection1/data
   [junit4]   2> 50199 T124 [collection1] [] [] [collection1]  oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 50200 T124 [collection1] [] [] [collection1]  oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 50200 T124 [collection1] [] [] [collection1]  oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 9A810A1455DE44C8-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 50201 T124 [collection1] [] [] [collection1]  oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=12, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.24691946960350455]
   [junit4]   2> 50203 T124 [collection1] [] [] [collection1]  oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@7bb0a884 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@b07b18d; maxCacheMB=0.66015625 maxMergeSizeMB=0.83203125)),segFN=segments_1,generation=1}
   [junit4]   2> 50203 T124 [collection1] [] [] [collection1]  oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 50220 T124 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 50221 T124 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 50221 T124 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 50222 T124 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 50222 T124 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 50223 T124 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 50224 T124 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 50225 T124 [collection1] [] [] [collection1]  oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 50225 T124 [collection1] [] [] [collection1]  oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 50258 T124 [collection1] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 50263 T124 [collection1] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 50268 T124 [collection1] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 50272 T124 [collection1] [] [] [collection1]  oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 50318 T124 [collection1] [] [] [collection1]  oasc.RequestHandlers.initHandlersFromConfig Registered paths: standard,/admin/system,/update/csv,/admin/luke,/admin/properties,/schema,/replication,/get,/update/json/docs,/config,/update/json,/admin/plugins,/update,/admin/segments,/admin/threads,/admin/logging,/admin/ping,/admin/file,/admin/mbeans
   [junit4]   2> 50322 T124 [collection1] [] [] [collection1]  oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 50324 T124 [collection1] [] [] [collection1]  oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 50325 T124 [collection1] [] [] [collection1]  oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 50327 T124 [collection1] [] [] [collection1]  oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 50328 T124 [collection1] [] [] [collection1]  oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 50330 T124 [collection1] [] [] [collection1]  oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=874432736, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 50332 T124 [collection1] [] [] [collection1]  oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@7bb0a884 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@b07b18d; maxCacheMB=0.66015625 maxMergeSizeMB=0.83203125)),segFN=segments_1,generation=1}
   [junit4]   2> 50332 T124 [collection1] [] [] [collection1]  oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 50333 T124 [collection1] [] [] [collection1]  oass.SolrIndexSearcher.<init> Opening Searcher@3bfa2596[collection1] main
   [junit4]   2> 50333 T124 [collection1] [] [] [collection1]  oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 50336 T124 [collection1] [] [] [collection1]  oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 50337 T124 [collection1] [] [] [collection1]  oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 50338 T124 [collection1] [] [] [collection1]  oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 50338 T124 [collection1] [] [] [collection1]  oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 50339 T124 [collection1] [] [] [collection1]  oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 50340 T124 [collection1] [] [] [collection1]  oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 50341 T124 [collection1] [] [] [collection1]  oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 50342 T124 [collection1] [] [] [collection1]  oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 50342 T124 [collection1] [] [] [collection1]  oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 50347 T125 [collection1] [] [] [collection1]  oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3bfa2596[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 50349 T124 [collection1] [] [] [collection1]  oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 50350 T128 [collection1] [shard1] [] []  oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:38885 collection:collection1 shard:shard1
   [junit4]   2> 50351 T67 [] [] [] []  oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 50352 T67 [] [] [] []  oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 50353 T128 [collection1] [shard1] [] []  oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 50365 T128 [collection1] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 50368 T93 [] [] [] []  oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 50369 T128 [collection1] [shard1] [] []  oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 50370 T128 [collection1] [shard1] [] []  oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C5 name=collection1 org.apache.solr.core.SolrCore@705c1320 url=http://127.0.0.1:38885/collection1 node=127.0.0.1:38885_ C5_STATE=coll:collection1 core:collection1 props:{node_name=127.0.0.1:38885_, state=down, core=collection1, base_url=http://127.0.0.1:38885}
   [junit4]   2> 50370 T128 [collection1] [shard1] [] []  C5 P38885 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:38885/collection1/
   [junit4]   2> 50371 T128 [collection1] [shard1] [] []  C5 P38885 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 503

[...truncated too long message...]

0 connections to: http://127.0.0.1:38284/
   [junit4]   2> 2367530 T9688 [control_collection] [shard1] [] [collection1]  oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:38293/
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=9A810A1455DE44C8 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=fi -Dtests.timezone=Europe/Ljubljana -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   79.3s J2 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18764/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([9A810A1455DE44C8:12D535CEFB222930]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18764/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	... 53 more
   [junit4]   2> 2367561 T9688 [control_collection] [shard1] [] [collection1]  oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest 9A810A1455DE44C8-001
   [junit4]   2> 79354 T9687 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {range_facet_l=PostingsFormat(name=Memory doPackFST= false), text=PostingsFormat(name=LuceneVarGapDocFreqInterval), timestamp=PostingsFormat(name=Memory doPackFST= false), multiDefault=PostingsFormat(name=Memory doPackFST= true), a_t=PostingsFormat(name=Memory doPackFST= false), _version_=PostingsFormat(name=Memory doPackFST= false), intDefault=PostingsFormat(name=Memory doPackFST= false), id=FSTOrd50}, docValues:{timestamp=DocValuesFormat(name=Memory)}, sim=DefaultSimilarity, locale=fi, timezone=Europe/Ljubljana
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=58507888,total=297271296
   [junit4]   2> NOTE: All tests run in this JVM: [BadCopyFieldTest, SynonymTokenizerTest, NotRequiredUniqueKeyTest, SyncSliceTest, TestPerFieldSimilarity, URLClassifyProcessorTest, DirectUpdateHandlerTest, DocumentBuilderTest, SOLR749Test, TestExtendedDismaxParser, RollingRestartTest, BadComponentTest, ReturnFieldsTest, SSLMigrationTest, TestExactStatsCache, TestNonDefinedSimilarityFactory, InfoHandlerTest, TestLRUCache, TestReplicationHandlerBackup, SaslZkACLProviderTest, CacheHeaderTest, TestSolrQueryParser, TestBlobHandler, DeleteReplicaTest, SimpleMLTQParserTest, AnalyticsQueryTest, TestManagedSchemaDynamicFieldResource, HdfsDirectoryTest, TestUniqueKeyFieldResource, SolrRequestParserTest, DirectUpdateHandlerOptimizeTest, TestFastWriter, TestReloadAndDeleteDocs, SolrPluginUtilsTest, TestRTGBase, DateFieldTest, ActionThrottleTest, TestFieldSortValues, HdfsChaosMonkeySafeLeaderTest, SliceStateTest, TestRemoteStreaming, TestRandomMergePolicy, DistributedIntervalFacetingTest, DistributedQueueTest, TestCSVResponseWriter, HardAutoCommitTest, TestHashPartitioner, DistribCursorPagingTest, TestReload, TestObjectReleaseTracker, TestExceedMaxTermLength, SolrCoreCheckLockOnStartupTest, TestRealTimeGet, TestBM25SimilarityFactory, QueryEqualityTest, TestMergePolicyConfig, LukeRequestHandlerTest, ChaosMonkeyNothingIsSafeTest, CurrencyFieldOpenExchangeTest, TestFoldingMultitermQuery, SpatialFilterTest, PolyFieldTest, WordBreakSolrSpellCheckerTest, TestAtomicUpdateErrorCases, DistanceFunctionTest, XsltUpdateRequestHandlerTest, TestWriterPerf, PrimitiveFieldTypeTest, TermsComponentTest, MoreLikeThisHandlerTest, RegexBoostProcessorTest, MBeansHandlerTest, TestPHPSerializedResponseWriter, TestComponentsName, TestLFUCache, TestSolrIndexConfig, MultiTermTest, MinimalSchemaTest, TestFuzzyAnalyzedSuggestions, TestLMDirichletSimilarityFactory, TimeZoneUtilsTest, PrimUtilsTest, SpellingQueryConverterTest, TestSolrJ, DistributedMLTComponentTest, AsyncMigrateRouteKeyTest, DistribDocExpirationUpdateProcessorTest, MigrateRouteKeyTest, ReplicationFactorTest]
   [junit4] Completed on J2 in 80.14s, 1 test, 1 error <<< FAILURES!

[...truncated 328 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 475 suites, 1894 tests, 3 errors, 58 ignored (9 assumptions)

Total time: 225 minutes 9 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 817761489 bytes
Compression is 0.0%
Took 2 min 32 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 796 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/796/

4 tests failed.
REGRESSION:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1427211454955,generation=2,filelist=[_81.fdt, _81.fdx, _81.fnm, _81.nvd, _81.nvm, _81.si, _81_Asserting_0.doc, _81_Asserting_0.tim, _81_Asserting_0.tip, _86.fdt, _86.fdx, _86.fnm, _86.nvd, _86.nvm, _86.si, _86_Asserting_0.doc, _86_Asserting_0.tim, _86_Asserting_0.tip, _88.fdt, _88.fdx, _88.fnm, _88.nvd, _88.nvm, _88.si, _88_Asserting_0.doc, _88_Asserting_0.tim, _88_Asserting_0.tip, _8a.fdt, _8a.fdx, _8a.fnm, _8a.nvd, _8a.nvm, _8a.si, _8a_Asserting_0.doc, _8a_Asserting_0.tim, _8a_Asserting_0.tip, _8b.cfe, _8b.cfs, _8b.si, segments_2]}]> but was:<[{indexVersion=1427211454955,generation=2,filelist=[_81.fdt, _81.fdx, _81.fnm, _81.nvd, _81.nvm, _81.si, _81_Asserting_0.doc, _81_Asserting_0.tim, _81_Asserting_0.tip, _86.fdt, _86.fdx, _86.fnm, _86.nvd, _86.nvm, _86.si, _86_Asserting_0.doc, _86_Asserting_0.tim, _86_Asserting_0.tip, _88.fdt, _88.fdx, _88.fnm, _88.nvd, _88.nvm, _88.si, _88_Asserting_0.doc, _88_Asserting_0.tim, _88_Asserting_0.tip, _8a.fdt, _8a.fdx, _8a.fnm, _8a.nvd, _8a.nvm, _8a.si, _8a_Asserting_0.doc, _8a_Asserting_0.tim, _8a_Asserting_0.tip, _8b.cfe, _8b.cfs, _8b.si, segments_2]}, {indexVersion=1427211454955,generation=3,filelist=[_81.fdt, _81.fdx, _81.fnm, _81.nvd, _81.nvm, _81.si, _81_Asserting_0.doc, _81_Asserting_0.tim, _81_Asserting_0.tip, _86.fdt, _86.fdx, _86.fnm, _86.nvd, _86.nvm, _86.si, _86_Asserting_0.doc, _86_Asserting_0.tim, _86_Asserting_0.tip, _8c.cfe, _8c.cfs, _8c.si, segments_3]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1427211454955,generation=2,filelist=[_81.fdt, _81.fdx, _81.fnm, _81.nvd, _81.nvm, _81.si, _81_Asserting_0.doc, _81_Asserting_0.tim, _81_Asserting_0.tip, _86.fdt, _86.fdx, _86.fnm, _86.nvd, _86.nvm, _86.si, _86_Asserting_0.doc, _86_Asserting_0.tim, _86_Asserting_0.tip, _88.fdt, _88.fdx, _88.fnm, _88.nvd, _88.nvm, _88.si, _88_Asserting_0.doc, _88_Asserting_0.tim, _88_Asserting_0.tip, _8a.fdt, _8a.fdx, _8a.fnm, _8a.nvd, _8a.nvm, _8a.si, _8a_Asserting_0.doc, _8a_Asserting_0.tim, _8a_Asserting_0.tip, _8b.cfe, _8b.cfs, _8b.si, segments_2]}]> but was:<[{indexVersion=1427211454955,generation=2,filelist=[_81.fdt, _81.fdx, _81.fnm, _81.nvd, _81.nvm, _81.si, _81_Asserting_0.doc, _81_Asserting_0.tim, _81_Asserting_0.tip, _86.fdt, _86.fdx, _86.fnm, _86.nvd, _86.nvm, _86.si, _86_Asserting_0.doc, _86_Asserting_0.tim, _86_Asserting_0.tip, _88.fdt, _88.fdx, _88.fnm, _88.nvd, _88.nvm, _88.si, _88_Asserting_0.doc, _88_Asserting_0.tim, _88_Asserting_0.tip, _8a.fdt, _8a.fdx, _8a.fnm, _8a.nvd, _8a.nvm, _8a.si, _8a_Asserting_0.doc, _8a_Asserting_0.tim, _8a_Asserting_0.tip, _8b.cfe, _8b.cfs, _8b.si, segments_2]}, {indexVersion=1427211454955,generation=3,filelist=[_81.fdt, _81.fdx, _81.fnm, _81.nvd, _81.nvm, _81.si, _81_Asserting_0.doc, _81_Asserting_0.tim, _81_Asserting_0.tip, _86.fdt, _86.fdx, _86.fnm, _86.nvd, _86.nvm, _86.si, _86_Asserting_0.doc, _86_Asserting_0.tim, _86_Asserting_0.tip, _8c.cfe, _8c.cfs, _8c.si, segments_3]}]>
	at __randomizedtesting.SeedInfo.seed([1B328619A7B3AADD:3EE59D29D7FBA4DE]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1137)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:33777/feox/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33777/feox/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([1B328619A7B3AADD:9366B9C3094FC725]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([1B328619A7B3AADD:9366B9C3094FC725]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([1B328619A7B3AADD:9366B9C3094FC725]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 10277 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/init-core-data-001
   [junit4]   2> 864097 T7305 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 864103 T7305 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 864104 T7306 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 864204 T7305 oasc.ZkTestServer.run start zk server on port:37206
   [junit4]   2> 864205 T7305 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 864206 T7305 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 864214 T7313 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@529f7661 name:ZooKeeperConnection Watcher:127.0.0.1:37206 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 864215 T7305 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 864215 T7305 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 864215 T7305 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 864229 T7305 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 864230 T7305 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 864232 T7316 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1c45162 name:ZooKeeperConnection Watcher:127.0.0.1:37206/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 864232 T7305 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 864232 T7305 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 864232 T7305 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 864237 T7305 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 864238 T7305 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 864239 T7305 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 864241 T7305 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 864244 T7305 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 864246 T7305 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 864247 T7305 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 864349 T7305 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 864350 T7305 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 864351 T7305 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 864352 T7305 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 864353 T7305 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 864354 T7305 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 864356 T7305 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 864356 T7305 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 864358 T7305 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 864359 T7305 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 864360 T7305 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 864361 T7305 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 864363 T7305 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 864363 T7305 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 864365 T7305 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 864366 T7305 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 864367 T7305 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 864368 T7305 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 865536 T7305 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/control-001/cores/collection1
   [junit4]   2> 865539 T7305 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 865552 T7305 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:52967
   [junit4]   2> 865552 T7305 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/tempDir-001/control/data, hostContext=/, hostPort=49330}
   [junit4]   2> 865552 T7305 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 865553 T7305 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/control-001/'
   [junit4]   2> 865591 T7305 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/control-001/solr.xml
   [junit4]   2> 865629 T7305 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/control-001/cores
   [junit4]   2> 865629 T7305 oasc.CoreContainer.<init> New CoreContainer 417507868
   [junit4]   2> 865629 T7305 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/control-001/]
   [junit4]   2> 865629 T7305 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/control-001/lib
   [junit4]   2> 865630 T7305 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/control-001/lib).
   [junit4]   2> 865641 T7305 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 865641 T7305 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 865641 T7305 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 865641 T7305 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 865642 T7305 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 865642 T7305 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 865642 T7305 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 865642 T7305 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 865642 T7305 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 865643 T7305 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 865643 T7305 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 865643 T7305 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 865643 T7305 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 865644 T7305 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 865644 T7305 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 865644 T7305 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:37206/solr
   [junit4]   2> 865646 T7305 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 865647 T7305 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 865647 T7305 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 865649 T7328 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@412f4820 name:ZooKeeperConnection Watcher:127.0.0.1:37206 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 865651 T7305 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 865652 T7305 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 865658 T7305 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 865659 T7331 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@c32281 name:ZooKeeperConnection Watcher:127.0.0.1:37206/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 865659 T7305 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 865662 T7305 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 865663 T7305 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 865665 T7305 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 865666 T7305 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 865668 T7305 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 865669 T7305 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 865671 T7305 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 865672 T7305 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 865672 T7305 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49330_
   [junit4]   2> 865673 T7305 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49330_
   [junit4]   2> 865674 T7305 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 865675 T7305 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 865676 T7305 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 865677 T7305 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:49330_
   [junit4]   2> 865677 T7305 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 865678 T7305 oasc.Overseer.start Overseer (id=93533682690162691-127.0.0.1:49330_-n_0000000000) starting
   [junit4]   2> 865680 T7305 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 865684 T7305 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 865684 T7333 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 865685 T7332 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 865684 T7305 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 865690 T7305 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/control-001/cores
   [junit4]   2> 865692 T7305 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {dataDir=data/, transient=false, collection=control_collection, coreNodeName=, schema=schema.xml, shard=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/control-001/cores/collection1, loadOnStartup=true, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/control-001/cores/collection1/, name=collection1}
   [junit4]   2> 865692 T7305 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/control-001/cores/collection1/
   [junit4]   2> 865692 T7305 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 865693 T7335 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 865693 T7335 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 865694 T7335 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 865694 T7331 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 865694 T7332 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49330",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:49330_"} current state version: 0
   [junit4]   2> 865695 T7332 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49330",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:49330_"}
   [junit4]   2> 865695 T7332 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 865697 T7332 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 865699 T7331 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 866694 T7335 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 866695 T7335 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 866695 T7335 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 866695 T7335 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 866696 T7335 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 866696 T7335 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/control-001/cores/collection1/'
   [junit4]   2> 866713 T7335 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 866715 T7335 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 866722 T7335 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 866738 T7335 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 866799 T7335 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 866900 T7335 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 866910 T7335 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 867321 T7335 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 867336 T7335 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 867339 T7335 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 867349 T7335 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 867354 T7335 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 867357 T7335 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 867358 T7335 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 867358 T7335 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 867358 T7335 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 867359 T7335 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 867359 T7335 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 867359 T7335 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 867359 T7335 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 867360 T7335 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/control-001/cores/collection1/data/
   [junit4]   2> 867360 T7335 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1c51fb0b
   [junit4]   2> 867362 T7335 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/control-001/cores/collection1/data
   [junit4]   2> 867363 T7335 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/control-001/cores/collection1/data/index/
   [junit4]   2> 867363 T7335 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 867363 T7335 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/control-001/cores/collection1/data/index
   [junit4]   2> 867364 T7335 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=49, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 867365 T7335 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@251b1137 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@140744e9),segFN=segments_1,generation=1}
   [junit4]   2> 867365 T7335 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 867370 T7335 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 867371 T7335 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 867371 T7335 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 867371 T7335 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 867371 T7335 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 867372 T7335 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 867372 T7335 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 867372 T7335 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 867373 T7335 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 867391 T7335 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 867394 T7335 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 867396 T7335 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 867398 T7335 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 867403 T7335 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,standard,/admin/ping,/admin/segments,/admin/logging,/update/json/docs,/get,/update/csv,/admin/luke,/replication,/update/json,/admin/file,/config,/admin/threads,/admin/mbeans,/admin/properties,/admin/plugins,/update,/schema
   [junit4]   2> 867405 T7335 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 867406 T7335 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 867406 T7335 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 867408 T7335 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 867408 T7335 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 867409 T7335 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=40, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=52.0576171875, floorSegmentMB=1.57421875, forceMergeDeletesPctAllowed=7.0197366319197485, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1836707464825394
   [junit4]   2> 867409 T7335 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@251b1137 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@140744e9),segFN=segments_1,generation=1}
   [junit4]   2> 867409 T7335 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 867410 T7335 oass.SolrIndexSearcher.<init> Opening Searcher@2bc74568[collection1] main
   [junit4]   2> 867410 T7335 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 867411 T7335 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 867411 T7335 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 867411 T7335 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 867412 T7335 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 867412 T7335 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 867412 T7335 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 867412 T7335 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 867413 T7335 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 867413 T7335 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 867414 T7336 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2bc74568[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 867414 T7335 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 867415 T7339 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:49330 collection:control_collection shard:shard1
   [junit4]   2> 867415 T7305 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 867415 T7305 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 867415 T7339 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 867418 T7305 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 867419 T7305 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 867419 T7339 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 867420 T7343 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3cb61cb7 name:ZooKeeperConnection Watcher:127.0.0.1:37206/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 867420 T7305 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 867420 T7331 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 867421 T7305 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 867421 T7339 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 867421 T7305 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 867421 T7332 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 867421 T7339 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C2699 name=collection1 org.apache.solr.core.SolrCore@15248747 url=http://127.0.0.1:49330/collection1 node=127.0.0.1:49330_ C2699_STATE=coll:control_collection core:collection1 props:{state=down, base_url=http://127.0.0.1:49330, core=collection1, node_name=127.0.0.1:49330_}
   [junit4]   2> 867424 T7339 C2699 P49330 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:49330/collection1/
   [junit4]   2> 867424 T7339 C2699 P49330 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 867425 T7339 C2699 P49330 oasc.SyncStrategy.syncToMe http://127.0.0.1:49330/collection1/ has no replicas
   [junit4]   2> 867425 T7339 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:49330/collection1/ shard1
   [junit4]   2> 867425 T7331 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 867425 T7339 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 867425 T7343 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 867429 T7305 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 867429 T7305 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 867431 T7305 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 867432 T7305 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 867433 T7346 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3b8c373a name:ZooKeeperConnection Watcher:127.0.0.1:37206/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 867433 T7305 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 867434 T7331 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 867434 T7305 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 867437 T7332 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49330",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 867441 T7332 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 867441 T7332 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 867442 T7331 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 867442 T7343 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 867444 T7331 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 867445 T7332 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 867445 T7331 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 867445 T7343 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 867484 T7339 oasc.ZkController.register We are http://127.0.0.1:49330/collection1/ and leader is http://127.0.0.1:49330/collection1/
   [junit4]   2> 867485 T7339 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:49330
   [junit4]   2> 867485 T7339 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 867485 T7339 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 867485 T7339 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 867486 T7331 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 867487 T7332 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49330",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:49330_"} current state version: 4
   [junit4]   2> 867487 T7332 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49330",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:49330_"}
   [junit4]   2> 867590 T7331 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 867590 T7343 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 868612 T7305 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/shard-1-001/cores/collection1
   [junit4]   2> 868614 T7305 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/shard-1-001
   [junit4]   2> 868615 T7305 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 868619 T7305 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:35295
   [junit4]   2> 868620 T7305 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/shard-1-001/cores, hostPort=10900, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/tempDir-001/jetty1}
   [junit4]   2> 868620 T7305 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 868621 T7305 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/shard-1-001/'
   [junit4]   2> 868652 T7305 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/shard-1-001/solr.xml
   [junit4]   2> 868687 T7305 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/shard-1-001/cores
   [junit4]   2> 868688 T7305 oasc.CoreContainer.<init> New CoreContainer 1317080909
   [junit4]   2> 868688 T7305 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/shard-1-001/]
   [junit4]   2> 868688 T7305 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/shard-1-001/lib
   [junit4]   2> 868689 T7305 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/shard-1-001/lib).
   [junit4]   2> 868715 T7305 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 868715 T7305 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 868715 T7305 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 868716 T7305 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 868716 T7305 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 868716 T7305 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 868717 T7305 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 868717 T7305 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 868717 T7305 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 868717 T7305 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 868718 T7305 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 868719 T7305 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 868719 T7305 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 868719 T7305 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 868719 T7305 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 868720 T7305 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:37206/solr
   [junit4]   2> 868720 T7305 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 868720 T7305 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 868721 T7305 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 868723 T7358 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@648ad2f1 name:ZooKeeperConnection Watcher:127.0.0.1:37206 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 868725 T7305 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 868726 T7305 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 868733 T7305 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 868734 T7361 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@12c9576c name:ZooKeeperConnection Watcher:127.0.0.1:37206/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 868734 T7305 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 868740 T7305 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 869743 T7305 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:10900_
   [junit4]   2> 869744 T7305 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:10900_
   [junit4]   2> 869749 T7305 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 869752 T7305 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/shard-1-001/cores
   [junit4]   2> 869754 T7305 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {coreNodeName=, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/shard-1-001/cores/collection1, shard=, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/shard-1-001/cores/collection1/, schema=schema.xml, dataDir=data/, transient=false, config=solrconfig.xml, collection=collection1}
   [junit4]   2> 869755 T7305 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/shard-1-001/cores/collection1/
   [junit4]   2> 869755 T7305 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 869757 T7362 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 869757 T7362 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 869758 T7331 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 869759 T7362 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 869759 T7362 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 869759 T7332 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:10900",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:10900_"} current state version: 5
   [junit4]   2> 869759 T7362 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 869759 T7332 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:10900",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:10900_"}
   [junit4]   2> 869760 T7332 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 869760 T7332 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 869760 T7362 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 869760 T7362 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 869761 T7332 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 869762 T7361 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 869762 T7361 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 870761 T7362 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 870762 T7362 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 870762 T7362 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 870762 T7362 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 870763 T7362 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 870763 T7362 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/shard-1-001/cores/collection1/'
   [junit4]   2> 870783 T7362 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 870786 T7362 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 870796 T7362 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 870818 T7362 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 870867 T7362 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 870968 T7362 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 870978 T7362 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 871378 T7362 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 871394 T7362 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 871397 T7362 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 871409 T7362 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 871413 T7362 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 871417 T7362 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 871418 T7362 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 871418 T7362 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 871418 T7362 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 871419 T7362 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 871419 T7362 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 871419 T7362 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 871419 T7362 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 871420 T7362 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 871420 T7362 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1c51fb0b
   [junit4]   2> 871422 T7362 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/shard-1-001/cores/collection1/data
   [junit4]   2> 871423 T7362 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 871423 T7362 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 871424 T7362 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 1B328619A7B3AADD-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 871424 T7362 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=49, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 871425 T7362 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1325a257 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@bc50793),segFN=segments_1,generation=1}
   [junit4]   2> 871425 T7362 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 871432 T7362 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 871432 T7362 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 871432 T7362 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 871433 T7362 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 871433 T7362 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 871433 T7362 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 871434 T7362 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 871434 T7362 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 871434 T7362 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 871453 T7362 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 871456 T7362 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 871458 T7362 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 871460 T7362 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 871467 T7362 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/system,standard,/admin/ping,/admin/segments,/admin/logging,/update/json/docs,/get,/update/csv,/admin/luke,/replication,/update/json,/admin/file,/config,/admin/threads,/admin/mbeans,/admin/properties,/admin/plugins,/update,/schema
   [junit4]   2> 871469 T7362 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 871470 T7362 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 871471 T7362 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 871472 T7362 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 871472 T7362 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 871473 T7362 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=40, maxMergeAtOnceExplicit=44, maxMergedSegmentMB=52.0576171875, floorSegmentMB=1.57421875, forceMergeDeletesPctAllowed=7.0197366319197485, segmentsPerTier=37.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1836707464825394
   [junit4]   2> 871474 T7362 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1325a257 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@bc50793),segFN=segments_1,generation=1}
   [junit4]   2> 871474 T7362 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 871474 T7362 oass.SolrIndexSearcher.<init> Opening Searcher@492db669[collection1] main
   [junit4]   2> 871475 T7362 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 871476 T7362 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 871476 T7362 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 871476 T7362 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 871476 T7362 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 871477 T7362 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 871477 T7362 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 871477 T7362 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 871478 T7362 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 871478 T7362 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 871479 T7363 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@492db669[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 871479 T7362 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 871480 T7366 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:10900 collection:collection1 shard:shard1
   [junit4]   2> 871480 T7305 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 871480 T7305 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 871481 T7366 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 871484 T7366 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 871486 T7331 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 871486 T7366 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 871486 T7366 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C2700 name=collection1 org.apache.solr.core.SolrCore@3347777e url=http://127.0.0.1:10900/collection1 node=127.0.0.1:10900_ C2700_STATE=coll:collection1 core:collection1 props:{state=down, base_url=http://127.0.0.1:10900, core=collection1, node_name=127.0.0.1:10900_}
   [junit4]   2> 871486 T7366 C2700 P10900 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:10900/collection1/
   [junit4]   2> 871486 T7366 C2700 P10900 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 871486 T7332 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 871486 T7366 C2700 P10900 oasc.SyncStrategy.syncToMe http://127.0.0.1:10900/collection1/ has no replicas
   [junit4]   2> 871487 T7366 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:10900/collection1/ shard1
   [junit4]   2> 871487 T7366 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 871487 T7332 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 871488 T7361 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 871489 T7361 oascc.ZkStateReader.updateWatchedCollec

[...truncated too long message...]

ect params={q=*:*&wt=javabin&version=2&sort=id+desc} hits=1 status=0 QTime=0 
   [junit4]   2> 1601099 T33592 oash.TestReplicationHandler.rQuery Waited for 1100ms and found 1 docs
   [junit4]   2> 1601102 T34505 C1510 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={_trace=getDetails&wt=javabin&command=details&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 1601105 T34505 C1510 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={_trace=getDetails&wt=javabin&command=details&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 1601111 T34507 C1510 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={slave=false&wt=javabin&command=details&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 1601112 T34517 C1512 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={_trace=getDetails&wt=javabin&command=details&version=2&qt=/replication} status=0 QTime=4 
   [junit4]   2> 1601118 T34505 C1510 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={_trace=assertVersions&wt=javabin&command=indexversion&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 1601121 T34517 C1512 oasc.SolrCore.execute [collection1] webapp=/solr path=/replication params={_trace=assertVersions&wt=javabin&command=indexversion&version=2&qt=/replication} status=0 QTime=0 
   [junit4]   2> 1601123 T33592 oas.SolrTestCaseJ4.tearDown ###Ending doTestIndexAndConfigReplication
   [junit4]   2> 1601123 T33592 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=375976131
   [junit4]   2> 1601124 T33592 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@68166d3a
   [junit4]   2> 1601125 T33592 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=1,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=1,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 1601126 T33592 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 1601126 T33592 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 1601126 T33592 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 1601127 T33592 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 1601128 T33592 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 1601128 T33592 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler 1B328619A7B3AADD-001/solr-instance-027/./collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler 1B328619A7B3AADD-001/solr-instance-027/./collection1/data;done=false>>]
   [junit4]   2> 1601129 T33592 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler 1B328619A7B3AADD-001/solr-instance-027/./collection1/data
   [junit4]   2> 1601129 T33592 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler 1B328619A7B3AADD-001/solr-instance-027/./collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler 1B328619A7B3AADD-001/solr-instance-027/./collection1/data/index;done=false>>]
   [junit4]   2> 1601130 T33592 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler 1B328619A7B3AADD-001/solr-instance-027/./collection1/data/index
   [junit4]   2> 1601131 T33592 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 1601327 T33592 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=62529800
   [junit4]   2> 1601328 T33592 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@6f46c663
   [junit4]   2> 1601329 T33592 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 1601329 T33592 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 1601330 T33592 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 1601330 T33592 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 1601331 T33592 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 1601332 T33592 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 1601332 T33592 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler 1B328619A7B3AADD-001/solr-instance-028/./collection1/data/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler 1B328619A7B3AADD-001/solr-instance-028/./collection1/data/index;done=false>>]
   [junit4]   2> 1601333 T33592 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler 1B328619A7B3AADD-001/solr-instance-028/./collection1/data/index
   [junit4]   2> 1601333 T33592 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler 1B328619A7B3AADD-001/solr-instance-028/./collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler 1B328619A7B3AADD-001/solr-instance-028/./collection1/data;done=false>>]
   [junit4]   2> 1601334 T33592 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler 1B328619A7B3AADD-001/solr-instance-028/./collection1/data
   [junit4]   2> 1601335 T33592 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/solr,null}
   [junit4]   2> 1601532 T33592 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.handler.TestReplicationHandler 1B328619A7B3AADD-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50), sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=lv, timezone=Etc/GMT-0
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=97708664,total=251658240
   [junit4]   2> NOTE: All tests run in this JVM: [TestDistributedSearch, DirectUpdateHandlerOptimizeTest, MergeStrategyTest, InfoHandlerTest, AnalyticsQueryTest, TestFieldSortValues, DistributedFacetPivotLongTailTest, TestSearcherReuse, SolrIndexSplitterTest, TestHashPartitioner, TestRandomDVFaceting, TestReplicationHandler]
   [junit4] Completed on J2 in 813.79s, 13 tests, 1 failure <<< FAILURES!

[...truncated 912 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 475 suites, 1893 tests, 3 errors, 1 failure, 58 ignored (9 assumptions)

Total time: 152 minutes 15 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 871179491 bytes
Compression is 0.0%
Took 2 min 30 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 795 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/795/

5 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test

Error Message:
Test abandoned because suite timeout was reached.

Stack Trace:
java.lang.Exception: Test abandoned because suite timeout was reached.
	at __randomizedtesting.SeedInfo.seed([CDB8CA2643962592]:0)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeySafeLeaderTest

Error Message:
Suite timeout exceeded (>= 7200000 msec).

Stack Trace:
java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
	at __randomizedtesting.SeedInfo.seed([CDB8CA2643962592]:0)


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:13179/_hfg/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:13179/_hfg/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([CDB8CA2643962592:45ECF5FCED6A486A]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([CDB8CA2643962592:45ECF5FCED6A486A]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([CDB8CA2643962592:45ECF5FCED6A486A]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 10013 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/init-core-data-001
   [junit4]   2> 297034 T1226 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 297042 T1226 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 297043 T1227 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 297143 T1226 oasc.ZkTestServer.run start zk server on port:55169
   [junit4]   2> 297144 T1226 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 297145 T1226 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 297149 T1234 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@245a9ceb name:ZooKeeperConnection Watcher:127.0.0.1:55169 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 297150 T1226 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 297150 T1226 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 297150 T1226 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 297154 T1226 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 297155 T1226 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 297156 T1237 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6113c967 name:ZooKeeperConnection Watcher:127.0.0.1:55169/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 297156 T1226 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 297157 T1226 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 297157 T1226 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 297159 T1226 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 297161 T1226 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 297162 T1226 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 297163 T1226 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 297164 T1226 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 297167 T1226 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 297167 T1226 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 297169 T1226 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 297170 T1226 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 297171 T1226 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 297171 T1226 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 297173 T1226 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 297173 T1226 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 297174 T1226 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 297175 T1226 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 297176 T1226 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 297177 T1226 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 297178 T1226 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 297178 T1226 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 297180 T1226 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 297180 T1226 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 297181 T1226 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 297182 T1226 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 297183 T1226 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 297184 T1226 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 298279 T1226 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/control-001/cores/collection1
   [junit4]   2> 298284 T1226 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 298289 T1226 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:55175
   [junit4]   2> 298289 T1226 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/control-001/cores, hostPort=55174, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/tempDir-001/control/data}
   [junit4]   2> 298289 T1226 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 298290 T1226 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/control-001/'
   [junit4]   2> 298323 T1226 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/control-001/solr.xml
   [junit4]   2> 298363 T1226 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/control-001/cores
   [junit4]   2> 298364 T1226 oasc.CoreContainer.<init> New CoreContainer 852829178
   [junit4]   2> 298364 T1226 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/control-001/]
   [junit4]   2> 298364 T1226 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/control-001/lib
   [junit4]   2> 298365 T1226 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/control-001/lib).
   [junit4]   2> 298374 T1226 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 298374 T1226 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 298374 T1226 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 298375 T1226 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 298375 T1226 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 298375 T1226 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 298375 T1226 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 298375 T1226 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 298376 T1226 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 298376 T1226 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 298376 T1226 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 298377 T1226 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 298377 T1226 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 298377 T1226 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 298377 T1226 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 298377 T1226 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:55169/solr
   [junit4]   2> 298378 T1226 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 298378 T1226 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 298379 T1226 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 298380 T1249 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4786ddf4 name:ZooKeeperConnection Watcher:127.0.0.1:55169 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 298380 T1226 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 298381 T1226 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 298384 T1226 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 298385 T1252 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7b0ca714 name:ZooKeeperConnection Watcher:127.0.0.1:55169/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 298385 T1226 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 298387 T1226 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 298389 T1226 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 298390 T1226 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 298392 T1226 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 298393 T1226 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 298395 T1226 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 298396 T1226 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 298397 T1226 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 298398 T1226 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55174_
   [junit4]   2> 298398 T1226 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:55174_
   [junit4]   2> 298399 T1226 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 298400 T1226 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 298402 T1226 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 298403 T1226 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:55174_
   [junit4]   2> 298403 T1226 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 298404 T1226 oasc.Overseer.start Overseer (id=93528013097271299-127.0.0.1:55174_-n_0000000000) starting
   [junit4]   2> 298405 T1226 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 298410 T1226 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 298411 T1254 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 298411 T1226 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 298411 T1253 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 298415 T1226 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/control-001/cores
   [junit4]   2> 298416 T1226 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/control-001/cores/collection1, collection=control_collection, schema=schema.xml, config=solrconfig.xml, coreNodeName=, name=collection1, shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/control-001/cores/collection1/, dataDir=data/, transient=false}
   [junit4]   2> 298417 T1226 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/control-001/cores/collection1/
   [junit4]   2> 298417 T1226 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 298418 T1256 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 298418 T1256 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 298419 T1256 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 298419 T1252 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 298419 T1253 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:55174_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55174",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 298420 T1253 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:55174_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55174",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 298420 T1253 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 298420 T1253 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 298421 T1252 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 299419 T1256 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 299420 T1256 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 299420 T1256 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 299421 T1256 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 299421 T1256 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 299422 T1256 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/control-001/cores/collection1/'
   [junit4]   2> 299455 T1256 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 299459 T1256 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 299472 T1256 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 299496 T1256 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 299559 T1256 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 299561 T1256 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 299579 T1256 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 300456 T1256 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 300475 T1256 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 300478 T1256 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 300495 T1256 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 300500 T1256 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 300506 T1256 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 300507 T1256 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 300507 T1256 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 300508 T1256 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 300508 T1256 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 300509 T1256 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 300509 T1256 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 300509 T1256 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 300510 T1256 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/control-001/cores/collection1/data/
   [junit4]   2> 300510 T1256 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@628006f9
   [junit4]   2> 300511 T1256 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/control-001/cores/collection1/data
   [junit4]   2> 300511 T1256 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/control-001/cores/collection1/data/index/
   [junit4]   2> 300511 T1256 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 300512 T1256 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/control-001/cores/collection1/data/index
   [junit4]   2> 300512 T1256 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=39, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.40437806714071256]
   [junit4]   2> 300513 T1256 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@10152f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7dee59f2),segFN=segments_1,generation=1}
   [junit4]   2> 300513 T1256 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 300519 T1256 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 300519 T1256 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 300520 T1256 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 300520 T1256 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 300520 T1256 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 300520 T1256 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 300521 T1256 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 300521 T1256 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 300521 T1256 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 300542 T1256 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 300545 T1256 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 300547 T1256 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 300549 T1256 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 300555 T1256 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/luke,/update/json/docs,/update/json,/schema,/admin/properties,/get,/update/csv,/admin/plugins,/admin/segments,/admin/file,/replication,/update,standard,/admin/system,/config,/admin/mbeans,/admin/ping,/admin/threads,/admin/logging
   [junit4]   2> 300557 T1256 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 300559 T1256 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 300559 T1256 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 300561 T1256 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 300561 T1256 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 300562 T1256 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=19, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4921984196149023]
   [junit4]   2> 300562 T1256 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@10152f9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7dee59f2),segFN=segments_1,generation=1}
   [junit4]   2> 300563 T1256 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 300563 T1256 oass.SolrIndexSearcher.<init> Opening Searcher@3476def[collection1] main
   [junit4]   2> 300563 T1256 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 300564 T1256 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 300565 T1256 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 300565 T1256 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 300565 T1256 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 300566 T1256 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 300566 T1256 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 300567 T1256 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 300567 T1256 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 300567 T1256 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 300568 T1257 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3476def[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 300569 T1256 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 300569 T1260 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:55174 collection:control_collection shard:shard1
   [junit4]   2> 300570 T1226 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 300570 T1226 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 300571 T1260 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 300575 T1226 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 300575 T1226 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 300576 T1260 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 300577 T1264 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@71ec2e23 name:ZooKeeperConnection Watcher:127.0.0.1:55169/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 300578 T1226 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 300578 T1252 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 300579 T1260 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 300579 T1226 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 300579 T1260 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 300579 T1253 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 300579 T1226 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C300 name=collection1 org.apache.solr.core.SolrCore@10ced6e6 url=http://127.0.0.1:55174/collection1 node=127.0.0.1:55174_ C300_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:55174_, core=collection1, base_url=http://127.0.0.1:55174, state=down}
   [junit4]   2> 300580 T1260 C300 P55174 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:55174/collection1/
   [junit4]   2> 300582 T1260 C300 P55174 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 300582 T1260 C300 P55174 oasc.SyncStrategy.syncToMe http://127.0.0.1:55174/collection1/ has no replicas
   [junit4]   2> 300583 T1260 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:55174/collection1/ shard1
   [junit4]   2> 300583 T1252 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 300583 T1260 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 300583 T1264 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 300586 T1226 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 300589 T1252 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 300590 T1253 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55174",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 300693 T1264 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 300695 T1252 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 300720 T1260 oasc.ZkController.register We are http://127.0.0.1:55174/collection1/ and leader is http://127.0.0.1:55174/collection1/
   [junit4]   2> 300721 T1260 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:55174
   [junit4]   2> 300721 T1260 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 300721 T1260 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 300721 T1260 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 300723 T1252 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 300724 T1253 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:55174_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55174",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state"} current state version: 3
   [junit4]   2> 300725 T1253 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:55174_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55174",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 300829 T1252 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 300829 T1264 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 302119 T1226 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-1-001/cores/collection1
   [junit4]   2> 302122 T1226 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-1-001
   [junit4]   2> 302123 T1226 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 302126 T1226 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:55181
   [junit4]   2> 302127 T1226 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-1-001/cores, hostPort=55180, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/tempDir-001/jetty1, solrconfig=solrconfig.xml}
   [junit4]   2> 302127 T1226 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 302128 T1226 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-1-001/'
   [junit4]   2> 302169 T1226 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-1-001/solr.xml
   [junit4]   2> 302230 T1226 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-1-001/cores
   [junit4]   2> 302231 T1226 oasc.CoreContainer.<init> New CoreContainer 4659739
   [junit4]   2> 302231 T1226 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-1-001/]
   [junit4]   2> 302231 T1226 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-1-001/lib
   [junit4]   2> 302232 T1226 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-1-001/lib).
   [junit4]   2> 302243 T1226 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 302244 T1226 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 302244 T1226 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 302244 T1226 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 302245 T1226 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 302245 T1226 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 302245 T1226 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 302245 T1226 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 302246 T1226 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 302246 T1226 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 302246 T1226 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 302247 T1226 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 302247 T1226 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 302247 T1226 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 302248 T1226 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 302248 T1226 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:55169/solr
   [junit4]   2> 302248 T1226 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 302249 T1226 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 302249 T1226 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 302251 T1276 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@41895501 name:ZooKeeperConnection Watcher:127.0.0.1:55169 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 302252 T1226 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 302252 T1226 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 302255 T1226 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 302258 T1279 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3892d69a name:ZooKeeperConnection Watcher:127.0.0.1:55169/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 302258 T1226 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 302264 T1226 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 303268 T1226 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:55180_
   [junit4]   2> 303269 T1226 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:55180_
   [junit4]   2> 303272 T1226 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 303275 T1226 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-1-001/cores
   [junit4]   2> 303276 T1226 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-1-001/cores/collection1/, name=collection1, coreNodeName=, loadOnStartup=true, schema=schema.xml, transient=false, collection=collection1, dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-1-001/cores/collection1, config=solrconfig.xml, shard=}
   [junit4]   2> 303277 T1226 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-1-001/cores/collection1/
   [junit4]   2> 303277 T1226 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 303278 T1280 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 303278 T1280 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 303279 T1280 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 303279 T1252 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 303280 T1253 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:55180_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55180",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 303280 T1253 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:55180_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55180",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 303280 T1253 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 303281 T1253 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 303282 T1264 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 303282 T1252 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 303282 T1279 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 304279 T1280 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 304280 T1280 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 304280 T1280 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 304280 T1280 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 304281 T1280 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 304281 T1280 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-1-001/cores/collection1/'
   [junit4]   2> 304299 T1280 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 304301 T1280 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 304309 T1280 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 304325 T1280 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 304367 T1280 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 304369 T1280 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 304377 T1280 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 304768 T1280 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 304780 T1280 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 304782 T1280 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 304797 T1280 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 304801 T1280 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 304805 T1280 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 304806 T1280 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 304806 T1280 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 304806 T1280 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 304807 T1280 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 304807 T1280 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 304807 T1280 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 304807 T1280 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 304808 T1280 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 304808 T1280 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@628006f9
   [junit4]   2> 304809 T1280 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-1-001/cores/collection1/data
   [junit4]   2> 304809 T1280 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 304809 T1280 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 304809 T1280 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 304810 T1280 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=39, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.40437806714071256]
   [junit4]   2> 304810 T1280 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@34ecb345 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1fed9228),segFN=segments_1,generation=1}
   [junit4]   2> 304811 T1280 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 304816 T1280 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 304816 T1280 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 304816 T1280 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 304817 T1280 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 304817 T1280 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 304817 T1280 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 304817 T1280 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 304817 T1280 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 304818 T1280 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 304831 T1280 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 304832 T1280 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 304834 T1280 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 304836 T1280 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 304841 T1280 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/luke,/update/json/docs,/update/json,/schema,/admin/properties,/get,/update/csv,/admin/plugins,/admin/segments,/admin/file,/replication,/update,standard,/admin/system,/config,/admin/mbeans,/admin/ping,/admin/threads,/admin/logging
   [junit4]   2> 304843 T1280 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 304844 T1280 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 304844 T1280 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 304845 T1280 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 304845 T1280 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 304847 T1280 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=19, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4921984196149023]
   [junit4]   2> 304847 T1280 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@34ecb345 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1fed9228),segFN=segments_1,generation=1}
   [junit4]   2> 304847 T1280 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 304848 T1280 oass.SolrIndexSearcher.<init> Opening Searcher@53b55f41[collection1] main
   [junit4]   2> 304848 T1280 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 304849 T1280 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 304849 T1280 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 304849 T1280 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 304850 T1280 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 304850 T1280 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 304850 T1280 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 304851 T1280 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 304851 T1280 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 304851 T1280 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 304852 T1281 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@53b55f41[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 304852 T1280 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 304853 T1284 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:55180 collection:collection1 shard:shard1
   [junit4]   2> 304853 T1226 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 304854 T1284 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 304854 T1226 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 304858 T1284 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 304860 T1252 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 304860 T1284 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 304860 T1284 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C301 name=collection1 org.apache.solr.core.SolrCore@d5d35f2 url=http://127.0.0.1:55180/collection1 node=127.0.0.1:55180_ C301_STATE=coll:collection1 core:collection1 props:{node_name=127.0.0.1:55180_, core=collection1, base_url=http://127.0.0.1:55180, state=down}
   [junit4]   2> 304861 T1284 C301 P55180 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:55180/collection1/
   [junit4]   2> 304861 T1284 C301 P55180 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 304861 T1253 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 304861 T1284 C301 P55180 oasc.SyncStrategy.syncToMe http://127.0.0.1:55180/collection1/ has no replicas
   [junit4]   2> 304861 T1284 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:55180/collection1/ shard1
   [junit4]   2> 304862 T1284 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 304863 T1252 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 304863 T1264 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 304863 T1279 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 304867 T1253 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55180",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 304868 T1252 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 304869 T1252 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 304869 T1264 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 304869 T1279 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 304917 T1284 oasc.ZkController.register We are http://127.0.0.1:55180/collection1/ and leader is http://127.0.0.1:55180/collection1/
   [junit4]   2> 304918 T1284 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:55180
   [junit4]   2> 304918 T1284 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 304918 T1284 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 304918 T1284 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 304920 T1252 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 304921 T1253 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "node_name":"127.0.0.1:55180_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55180",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state"} current state version: 7
   [junit4]   2> 304921 T1253 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "node_name":"127.0.0.1:55180_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:55180",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 305024 T1279 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 305024 T1264 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 305024 T1252 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 306109 T1226 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-2-001/cores/collection1
   [junit4]   2> 306112 T1226 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-2-001
   [junit4]   2> 306115 T1226 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 306118 T1226 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:55189
   [junit4]   2> 306118 T1226 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-2-001/cores, hostPort=55188, solrconfig=solrconfig.xml, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/tempDir-001/jetty2, hostContext=/}
   [junit4]   2> 306119 T1226 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 306119 T1226 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-2-001/'
   [junit4]   2> 306151 T1226 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-2-001/solr.xml
   [junit4]   2> 306190 T1226 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-2-001/cores
   [junit4]   2> 306190 T1226 oasc.CoreContainer.<init> New CoreContainer 548337341
   [junit4]   2> 306190 T1226 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-2-001/]
   [junit4]   2> 306191 T1226 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-2-001/lib
   [junit4]   2> 306191 T1226 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest CDB8CA2643962592-001/shard-2-001/lib).
   [junit4]   2> 306200 T1226 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 306200 T1226 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 306200 T1226 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 306201 T1226 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 306201 T1226 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 306201 T1226 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 306201 T1226 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 306202 T1226 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 306202 T1226 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 306202 T1226 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 306202 T1226 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 306203 T1226 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 306203 T1226 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 306203 T1226 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 306203 T1226 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 306204 T1226 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:55169/solr
   [junit4]   2> 306204 T1226 oasc.ZkController.checkChrootPath zkHost includes chroot
  

[...truncated too long message...]

dBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]   2> 	        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 	  93) Thread[id=2919, name=TEST-ChaosMonkeySafeLeaderTest.test-seed#[CDB8CA2643962592]-SendThread(127.0.0.1:38661), state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
   [junit4]   2> 	        at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:200)
   [junit4]   2> 	        at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:103)
   [junit4]   2> 	        at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:87)
   [junit4]   2> 	        at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:98)
   [junit4]   2> 	        at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349)
   [junit4]   2> 	        at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1081)
   [junit4]   2> 	  94) Thread[id=3696, name=coreZkRegister-2084-thread-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]   2> 	        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]   2> 	        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]   2> 	        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 	  95) Thread[id=2844, name=qtp1787110160-2844, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at java.net.SocketInputStream.socketRead0(Native Method)
   [junit4]   2> 	        at java.net.SocketInputStream.read(SocketInputStream.java:152)
   [junit4]   2> 	        at java.net.SocketInputStream.read(SocketInputStream.java:122)
   [junit4]   2> 	        at org.eclipse.jetty.io.ByteArrayBuffer.readFrom(ByteArrayBuffer.java:391)
   [junit4]   2> 	        at org.eclipse.jetty.io.bio.StreamEndPoint.fill(StreamEndPoint.java:141)
   [junit4]   2> 	        at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.fill(SocketConnector.java:227)
   [junit4]   2> 	        at org.eclipse.jetty.http.HttpParser.fill(HttpParser.java:1040)
   [junit4]   2> 	        at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:280)
   [junit4]   2> 	        at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
   [junit4]   2> 	        at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
   [junit4]   2> 	        at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
   [junit4]   2> 	        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]   2> 	        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]   2> 	        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 	  96) Thread[id=3269, name=searcherExecutor-1935-thread-1, state=WAITING, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at sun.misc.Unsafe.park(Native Method)
   [junit4]   2> 	        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
   [junit4]   2> 	        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
   [junit4]   2> 	        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
   [junit4]   2> 	        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]   2> 	        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 	  97) Thread[id=3415, name=qtp1895904405-3415, state=RUNNABLE, group=TGRP-ChaosMonkeySafeLeaderTest]
   [junit4]   2> 	        at java.net.SocketInputStream.socketRead0(Native Method)
   [junit4]   2> 	        at java.net.SocketInputStream.read(SocketInputStream.java:152)
   [junit4]   2> 	        at java.net.SocketInputStream.read(SocketInputStream.java:122)
   [junit4]   2> 	        at org.eclipse.jetty.io.ByteArrayBuffer.readFrom(ByteArrayBuffer.java:391)
   [junit4]   2> 	        at org.eclipse.jetty.io.bio.StreamEndPoint.fill(StreamEndPoint.java:141)
   [junit4]   2> 	        at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.fill(SocketConnector.java:227)
   [junit4]   2> 	        at org.eclipse.jetty.http.HttpParser.fill(HttpParser.java:1040)
   [junit4]   2> 	        at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:280)
   [junit4]   2> 	        at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:235)
   [junit4]   2> 	        at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
   [junit4]   2> 	        at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
   [junit4]   2> 	        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
   [junit4]   2> 	        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
   [junit4]   2> 	        at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {rnd_b=FST50, a_t=PostingsFormat(name=Asserting), id=FST50, a_i=FST50, _version_=PostingsFormat(name=Asserting)}, docValues:{}, sim=DefaultSimilarity, locale=es_HN, timezone=Africa/Maputo
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=105,free=57012568,total=418381824
   [junit4]   2> NOTE: All tests run in this JVM: [ZkControllerTest, TestRequestStatusCollectionAPI, TestConfig, TestQueryUtils, CustomCollectionTest, TestSortingResponseWriter, TestSerializedLuceneMatchVersion, CoreAdminCreateDiscoverTest, SuggesterFSTTest, HighlighterTest, TestCloudPivotFacet, JSONWriterTest, BlockDirectoryTest, TestJettySolrRunner, SortByFunctionTest, TestFieldResource, TestPhraseSuggestions, CopyFieldTest, TestBinaryResponseWriter, TestSolrConfigHandlerCloud, EnumFieldTest, TestSolrXml, UpdateParamsTest, SchemaVersionSpecificBehaviorTest, OpenCloseCoreStressTest, TestJsonFacets, QueryElevationComponentTest, DistributedFacetPivotSmallTest, AtomicUpdatesTest, TestSolr4Spatial, AutoCommitTest, HdfsWriteToMultipleCollectionsTest, TestDocumentBuilder, ConvertedLegacyTest, TestDefaultStatsCache, CircularListTest, SystemInfoHandlerTest, TestCoreDiscovery, DistributedQueryComponentCustomSortTest, TestNamedUpdateProcessors, ChaosMonkeySafeLeaderTest]
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ChaosMonkeySafeLeaderTest -Dtests.seed=CDB8CA2643962592 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=es_HN -Dtests.timezone=Africa/Maputo -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   0.00s J1 | ChaosMonkeySafeLeaderTest (suite) <<<
   [junit4]    > Throwable #1: java.lang.Exception: Suite timeout exceeded (>= 7200000 msec).
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([CDB8CA2643962592]:0)
   [junit4] Completed on J1 in 7225.58s, 1 test, 2 errors <<< FAILURES!

[...truncated 67 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 475 suites, 1892 tests, 1 suite-level error, 4 errors, 103 ignored (9 assumptions)

Total time: 241 minutes 32 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 808150801 bytes
Compression is 0.0%
Took 2 min 31 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 794 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/794/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:37879/_sey/kt/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37879/_sey/kt/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([2D5419922EBDFB88:A500264880419670]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([2D5419922EBDFB88:A500264880419670]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([2D5419922EBDFB88:A500264880419670]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 11017 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/init-core-data-001
   [junit4]   2> 2252512 T41381 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /_sey/kt
   [junit4]   2> 2252521 T41381 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 2252522 T41382 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 2252622 T41381 oasc.ZkTestServer.run start zk server on port:37845
   [junit4]   2> 2252623 T41381 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 2252624 T41381 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2252631 T41389 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@84b7085 name:ZooKeeperConnection Watcher:127.0.0.1:37845 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2252632 T41381 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2252632 T41381 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 2252632 T41381 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 2252640 T41381 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 2252641 T41381 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2252642 T41392 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6ffe912f name:ZooKeeperConnection Watcher:127.0.0.1:37845/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2252642 T41381 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2252643 T41381 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 2252643 T41381 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 2252645 T41381 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 2252646 T41381 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 2252648 T41381 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 2252649 T41381 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 2252650 T41381 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 2252653 T41381 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 2252653 T41381 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 2252756 T41381 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2252757 T41381 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 2252759 T41381 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 2252759 T41381 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 2252761 T41381 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 2252762 T41381 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 2252763 T41381 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 2252764 T41381 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 2252766 T41381 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 2252766 T41381 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 2252768 T41381 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2252769 T41381 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 2252770 T41381 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2252771 T41381 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 2252773 T41381 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 2252773 T41381 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 2252775 T41381 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 2252776 T41381 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 2254088 T41381 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/control-001/cores/collection1
   [junit4]   2> 2254092 T41381 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2254102 T41381 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:37850
   [junit4]   2> 2254102 T41381 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/tempDir-001/control/data, hostContext=/_sey/kt, hostPort=37849, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/control-001/cores}
   [junit4]   2> 2254102 T41381 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 2254103 T41381 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/control-001/'
   [junit4]   2> 2254136 T41381 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/control-001/solr.xml
   [junit4]   2> 2254188 T41381 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/control-001/cores
   [junit4]   2> 2254189 T41381 oasc.CoreContainer.<init> New CoreContainer 283581719
   [junit4]   2> 2254189 T41381 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/control-001/]
   [junit4]   2> 2254189 T41381 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/control-001/lib
   [junit4]   2> 2254190 T41381 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/control-001/lib).
   [junit4]   2> 2254198 T41381 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 2254198 T41381 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 2254199 T41381 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2254199 T41381 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2254199 T41381 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 2254199 T41381 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2254199 T41381 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2254200 T41381 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2254200 T41381 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2254200 T41381 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2254200 T41381 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 2254201 T41381 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2254201 T41381 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 2254201 T41381 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2254201 T41381 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 2254202 T41381 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:37845/solr
   [junit4]   2> 2254202 T41381 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2254202 T41381 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 2254203 T41381 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2254204 T41404 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@31cc2330 name:ZooKeeperConnection Watcher:127.0.0.1:37845 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2254204 T41381 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2254205 T41381 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 2254206 T41381 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2254207 T41407 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@67ede0ec name:ZooKeeperConnection Watcher:127.0.0.1:37845/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2254207 T41381 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2254208 T41381 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 2254209 T41381 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 2254211 T41381 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 2254212 T41381 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 2254213 T41381 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 2254214 T41381 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 2254215 T41381 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 2254216 T41381 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 2254217 T41381 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37849__sey%2Fkt
   [junit4]   2> 2254218 T41381 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:37849__sey%2Fkt
   [junit4]   2> 2254219 T41381 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 2254219 T41381 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 2254220 T41381 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 2254221 T41381 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:37849__sey%2Fkt
   [junit4]   2> 2254222 T41381 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 2254222 T41381 oasc.Overseer.start Overseer (id=93523755870519299-127.0.0.1:37849__sey%2Fkt-n_0000000000) starting
   [junit4]   2> 2254224 T41381 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 2254227 T41381 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 2254228 T41409 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 2254228 T41408 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 2254228 T41381 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2254232 T41381 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/control-001/cores
   [junit4]   2> 2254234 T41381 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {shard=, collection=control_collection, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/control-001/cores/collection1/, config=solrconfig.xml, coreNodeName=, loadOnStartup=true, dataDir=data/, schema=schema.xml, name=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/control-001/cores/collection1, transient=false}
   [junit4]   2> 2254234 T41381 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/control-001/cores/collection1/
   [junit4]   2> 2254234 T41381 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 2254235 T41411 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 2254235 T41411 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2254236 T41411 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2254236 T41407 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 2254237 T41408 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:37849__sey%2Fkt",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37849/_sey/kt"} current state version: 0
   [junit4]   2> 2254237 T41408 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:37849__sey%2Fkt",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37849/_sey/kt"}
   [junit4]   2> 2254237 T41408 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 2254237 T41408 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 2254238 T41407 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2255236 T41411 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 2255237 T41411 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 2255237 T41411 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 2255238 T41411 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 2255238 T41411 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2255238 T41411 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/control-001/cores/collection1/'
   [junit4]   2> 2255257 T41411 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 2255260 T41411 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 2255270 T41411 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 2255292 T41411 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 2255339 T41411 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2255441 T41411 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 2255451 T41411 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 2255873 T41411 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 2255914 T41411 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2255918 T41411 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2255926 T41411 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2255931 T41411 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2255935 T41411 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2255936 T41411 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2255936 T41411 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2255936 T41411 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2255937 T41411 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2255937 T41411 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2255937 T41411 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 2255937 T41411 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2255938 T41411 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/control-001/cores/collection1/data/
   [junit4]   2> 2255938 T41411 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5ff59ae3
   [junit4]   2> 2255940 T41411 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/control-001/cores/collection1/data
   [junit4]   2> 2255942 T41411 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/control-001/cores/collection1/data/index/
   [junit4]   2> 2255942 T41411 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 2255944 T41411 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/control-001/cores/collection1/data/index
   [junit4]   2> 2255944 T41411 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=17.509765625, floorSegmentMB=1.744140625, forceMergeDeletesPctAllowed=27.221643169507644, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 2255950 T41411 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/index-MMapDirectory-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@29f70c05),segFN=segments_1,generation=1}
   [junit4]   2> 2255951 T41411 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2255956 T41411 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 2255956 T41411 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 2255956 T41411 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 2255957 T41411 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2255957 T41411 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2255957 T41411 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 2255958 T41411 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2255958 T41411 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2255958 T41411 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 2255988 T41411 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2255991 T41411 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2255993 T41411 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2255996 T41411 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2256000 T41411 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /replication,/schema,standard,/update/json/docs,/admin/logging,/admin/ping,/admin/plugins,/config,/admin/properties,/update/json,/admin/segments,/get,/admin/mbeans,/update/csv,/admin/system,/update,/admin/file,/admin/threads,/admin/luke
   [junit4]   2> 2256003 T41411 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 2256005 T41411 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2256005 T41411 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 2256007 T41411 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2256007 T41411 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2256008 T41411 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=38, maxMergeAtOnceExplicit=32, maxMergedSegmentMB=14.8349609375, floorSegmentMB=1.4091796875, forceMergeDeletesPctAllowed=24.428034201313828, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 2256012 T41411 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/index-MMapDirectory-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@29f70c05),segFN=segments_1,generation=1}
   [junit4]   2> 2256012 T41411 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2256012 T41411 oass.SolrIndexSearcher.<init> Opening Searcher@6a14c52b[collection1] main
   [junit4]   2> 2256013 T41411 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 2256014 T41411 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2256014 T41411 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 2256014 T41411 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2256014 T41411 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 2256015 T41411 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2256015 T41411 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 2256015 T41411 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2256016 T41411 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 2256016 T41411 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 2256017 T41412 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6a14c52b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2256017 T41411 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 2256018 T41415 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:37849/_sey/kt collection:control_collection shard:shard1
   [junit4]   2> 2256018 T41381 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 2256018 T41381 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2256018 T41415 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 2256022 T41381 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 2256022 T41415 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 2256023 T41381 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2256024 T41407 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 2256024 T41419 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@411ed590 name:ZooKeeperConnection Watcher:127.0.0.1:37845/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2256024 T41415 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 2256025 T41381 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2256025 T41408 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 2256025 T41415 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 2256025 T41381 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> ASYNC  NEW_CORE C2480 name=collection1 org.apache.solr.core.SolrCore@29940184 url=http://127.0.0.1:37849/_sey/kt/collection1 node=127.0.0.1:37849__sey%2Fkt C2480_STATE=coll:control_collection core:collection1 props:{core=collection1, node_name=127.0.0.1:37849__sey%2Fkt, state=down, base_url=http://127.0.0.1:37849/_sey/kt}
   [junit4]   2> 2256026 T41415 C2480 P37849 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:37849/_sey/kt/collection1/
   [junit4]   2> 2256026 T41381 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2256026 T41407 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2256026 T41415 C2480 P37849 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 2256027 T41415 C2480 P37849 oasc.SyncStrategy.syncToMe http://127.0.0.1:37849/_sey/kt/collection1/ has no replicas
   [junit4]   2> 2256027 T41415 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:37849/_sey/kt/collection1/ shard1
   [junit4]   2> 2256027 T41415 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 2256029 T41381 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 2256029 T41381 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 2256029 T41381 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 2256030 T41381 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2256030 T41407 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 2256031 T41422 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6186145e name:ZooKeeperConnection Watcher:127.0.0.1:37845/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2256031 T41408 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37849/_sey/kt",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 2256031 T41381 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2256032 T41381 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 2256033 T41407 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 2256034 T41408 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 2256035 T41408 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 2256035 T41407 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2256035 T41419 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2256081 T41415 oasc.ZkController.register We are http://127.0.0.1:37849/_sey/kt/collection1/ and leader is http://127.0.0.1:37849/_sey/kt/collection1/
   [junit4]   2> 2256082 T41415 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:37849/_sey/kt
   [junit4]   2> 2256082 T41415 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 2256082 T41415 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 2256082 T41415 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2256083 T41407 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 2256084 T41408 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:37849__sey%2Fkt",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37849/_sey/kt"} current state version: 3
   [junit4]   2> 2256084 T41408 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:37849__sey%2Fkt",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37849/_sey/kt"}
   [junit4]   2> 2256085 T41408 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 2256086 T41407 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2256086 T41419 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2256189 T41407 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2256189 T41419 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 2257730 T41381 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/shard-1-001/cores/collection1
   [junit4]   2> 2257734 T41381 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/shard-1-001
   [junit4]   2> 2257734 T41381 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2257740 T41381 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:37880
   [junit4]   2> 2257740 T41381 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=37879, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/shard-1-001/cores, hostContext=/_sey/kt, solrconfig=solrconfig.xml}
   [junit4]   2> 2257741 T41381 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 2257741 T41381 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/shard-1-001/'
   [junit4]   2> 2257785 T41381 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/shard-1-001/solr.xml
   [junit4]   2> 2257833 T41381 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/shard-1-001/cores
   [junit4]   2> 2257833 T41381 oasc.CoreContainer.<init> New CoreContainer 2102715995
   [junit4]   2> 2257834 T41381 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/shard-1-001/]
   [junit4]   2> 2257834 T41381 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/shard-1-001/lib
   [junit4]   2> 2257834 T41381 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/shard-1-001/lib).
   [junit4]   2> 2257843 T41381 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 2257843 T41381 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 2257843 T41381 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 2257843 T41381 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 2257844 T41381 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 2257844 T41381 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 2257844 T41381 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 2257844 T41381 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 2257844 T41381 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 2257845 T41381 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 2257845 T41381 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 2257845 T41381 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 2257845 T41381 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 2257846 T41381 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 2257846 T41381 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 2257846 T41381 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:37845/solr
   [junit4]   2> 2257846 T41381 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 2257847 T41381 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 2257847 T41381 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2257849 T41434 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@164241bf name:ZooKeeperConnection Watcher:127.0.0.1:37845 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2257849 T41381 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2257849 T41381 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 2257851 T41381 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 2257852 T41437 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5fbd7043 name:ZooKeeperConnection Watcher:127.0.0.1:37845/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 2257852 T41381 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 2257855 T41381 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 2258858 T41381 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:37879__sey%2Fkt
   [junit4]   2> 2258859 T41381 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:37879__sey%2Fkt
   [junit4]   2> 2258864 T41381 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 2258866 T41381 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/shard-1-001/cores
   [junit4]   2> 2258869 T41381 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, shard=, transient=false, collection=collection1, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/shard-1-001/cores/collection1, config=solrconfig.xml, dataDir=data/, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/shard-1-001/cores/collection1/, schema=schema.xml}
   [junit4]   2> 2258869 T41381 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/shard-1-001/cores/collection1/
   [junit4]   2> 2258870 T41381 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 2258871 T41438 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 2258872 T41438 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2258873 T41407 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 2258873 T41438 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 2258873 T41438 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 2258873 T41408 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:37879__sey%2Fkt",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37879/_sey/kt"} current state version: 5
   [junit4]   2> 2258874 T41438 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 2258874 T41408 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:37879__sey%2Fkt",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37879/_sey/kt"}
   [junit4]   2> 2258874 T41408 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 2258874 T41408 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 2258875 T41438 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 2258875 T41438 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 2258876 T41408 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 2258876 T41437 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2258877 T41437 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 2259875 T41438 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 2259876 T41438 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 2259876 T41438 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 2259876 T41438 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 2259877 T41438 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2259877 T41438 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/shard-1-001/cores/collection1/'
   [junit4]   2> 2259890 T41438 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 2259892 T41438 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 2259899 T41438 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 2259915 T41438 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 2259948 T41438 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 2260049 T41438 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 2260056 T41438 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 2260334 T41438 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 2260344 T41438 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 2260346 T41438 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 2260352 T41438 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2260355 T41438 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 2260358 T41438 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2260358 T41438 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2260359 T41438 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2260359 T41438 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 2260359 T41438 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 2260359 T41438 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 2260359 T41438 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 2260360 T41438 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 2260360 T41438 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 2260360 T41438 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5ff59ae3
   [junit4]   2> 2260362 T41438 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/shard-1-001/cores/collection1/data
   [junit4]   2> 2260362 T41438 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 2260362 T41438 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 2260364 T41438 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 2260364 T41438 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=19, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=17.509765625, floorSegmentMB=1.744140625, forceMergeDeletesPctAllowed=27.221643169507644, segmentsPerTier=18.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 2260367 T41438 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/index-MMapDirectory-002 lockFactory=org.apache.lucene.store.NativeFSLockFactory@29f70c05),segFN=segments_1,generation=1}
   [junit4]   2> 2260367 T41438 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2260371 T41438 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 2260371 T41438 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 2260371 T41438 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 2260372 T41438 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2260372 T41438 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 2260372 T41438 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 2260372 T41438 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2260372 T41438 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 2260373 T41438 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 2260385 T41438 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2260386 T41438 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2260387 T41438 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2260389 T41438 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 2260394 T41438 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /replication,/schema,standard,/update/json/docs,/admin/logging,/admin/ping,/admin/plugins,/config,/admin/properties,/update/json,/admin/segments,/get,/admin/mbeans,/update/csv,/admin/system,/update,/admin/file,/admin/threads,/admin/luke
   [junit4]   2> 2260395 T41438 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 2260396 T41438 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 2260396 T41438 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 2260397 T41438 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 2260397 T41438 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 2260398 T41438 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=38, maxMergeAtOnceExplicit=32, maxMergedSegmentMB=14.8349609375, floorSegmentMB=1.4091796875, forceMergeDeletesPctAllowed=24.428034201313828, segmentsPerTier=12.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 2260400 T41438 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(MMapDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/index-MMapDirectory-002 lockFactory=org.apache.lucene.store.NativeFSLockFactory@29f70c05),segFN=segments_1,generation=1}
   [junit4]   2> 2260400 T41438 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 2260400 T41438 oass.SolrIndexSearcher.<init> Opening Searcher@68f9c421[collection1] main
   [junit4]   2> 2260401 T41438 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 2260401 T41438 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 2260401 T41438 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 2260402 T41438 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 2260402 T41438 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 2260402 T41438 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2260402 T41438 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 2260403 T41438 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 2260403 T41438 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 2260403 T41438 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 2260403 T41439 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@68f9c421[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 2260404 T41438 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 2260404 T41442 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:37879/_sey/kt collection:collection1 shard:shard1
   [junit4]   2> 2260404 T41381 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 2260405 T41381 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 2260405 T41442 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 2260407 T41442 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 2260408 T41407 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 2260408 T41442 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 2260409 T41442 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C2481 name=collection1 org.apache.solr.core.SolrCore@5caa62ac url=http://127.0.0.1:37879/_sey/kt/collection1 node=127.0.0.1:37879__sey%2Fkt C2481_STATE=coll:collection1 core:collection1 props:{core=collection1, node_name=127.0.0.1:37879__sey%2Fkt, state=down, base_url=http://127.0.0.1:37879/_sey/kt}
   [junit4]   2> 2260409 T41442 C2481 P37879 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:37879/_sey/kt/collection1/
   [junit4]   2> 2260409 T41408 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 2260409 T41442 C2481 P37879 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 2260409 T41442 C2481 P37879 oasc.SyncStrategy.syncToMe http://127.0.0.1:37879/_sey/kt/collection1/ has no replicas
   [junit4]   2> 2260409 T41442 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:37879/_sey/kt/collection1/ shard1
   [junit4]   2> 2260410 T41442 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 2260410 T41408 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 2260410 T41437 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2260411 T41437 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 2260412 T41407 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 2260412 T41408 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37879/_sey/kt",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 2260413 T41408 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 2260413 T41437 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2260414 T41437 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 2260462 T41442 oasc.ZkController.register We are http://127.0.0.1:37879/_sey/kt/collection1/ and leader is http://127.0.0.1:37879/_sey/kt/collection1/
   [junit4]   2> 2260463 T41442 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:37879/_sey/kt
   [junit4]   2> 2260463 T41442 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 2260463 T41442 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 2260463 T41442 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 2260464 T41407 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 2260464 T41408 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:37879__sey%2Fkt",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37879/_sey/kt"} current state version: 5
   [junit4]   2> 2260465 T41408 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:37879__sey%2Fkt",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:37879/_sey/kt"}
   [junit4]   2> 2260465 T41442 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 2260465 T41408 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 3
   [junit4]   2> 2260466 T41437 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 2260466 T41437 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 2261348 T41381 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/shard-2-001/cores/collection1
   [junit4]   2> 2261350 T41381 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/shard-2-001
   [junit4]   2> 2261351 T41381 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 2261356 T41381 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:37895
   [junit4]   2> 2261356 T41381 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=37894, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/tempDir-001/jetty2, hostContext=/_sey/kt, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2D5419922EBDFB88-001/shard-2-001/cores, solrconfig=solrconfig.xml}
   [junit4]   2> 2261357 T41381 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 2261357 T41381 oasc.SolrResourceLoader.<init> new SolrResourceLoader f

[...truncated too long message...]

BDFB88 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=en_AU -Dtests.timezone=Africa/Ouagadougou -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   72.3s J1 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([2D5419922EBDFB88:A500264880419670]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 2413869 T6921 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.ReplicationFactorTest 2D5419922EBDFB88-001
   [junit4]   2> 72313 T6920 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {intDefault=PostingsFormat(name=LuceneFixedGap), _version_=PostingsFormat(name=LuceneFixedGap), range_facet_l=PostingsFormat(name=LuceneFixedGap), timestamp=PostingsFormat(name=LuceneFixedGap), text=PostingsFormat(name=Asserting), id=PostingsFormat(name=MockRandom), a_t=PostingsFormat(name=LuceneFixedGap), multiDefault=PostingsFormat(name=LuceneVarGapDocFreqInterval)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {}, locale=en_AU, timezone=Africa/Ouagadougou
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=233689512,total=475004928
   [junit4]   2> NOTE: All tests run in this JVM: [AutoCommitTest, TestFieldResource, WordBreakSolrSpellCheckerTest, QueryElevationComponentTest, SystemInfoHandlerTest, ResourceLoaderTest, TestConfigReload, ChaosMonkeySafeLeaderTest, DistributedExpandComponentTest, TestHighFrequencyDictionaryFactory, TestCryptoKeys, TestDocBasedVersionConstraints, TestBadConfig, PreAnalyzedFieldTest, PluginInfoTest, TestStressVersions, SortByFunctionTest, TermVectorComponentTest, TestChildDocTransformer, FullSolrCloudDistribCmdsTest, TestAnalyzeInfixSuggestions, CollectionsAPIAsyncDistributedZkTest, AliasIntegrationTest, TestStressLucene, TestSweetSpotSimilarityFactory, TestDefaultStatsCache, DirectSolrConnectionTest, TestPhraseSuggestions, AtomicUpdatesTest, ResponseHeaderTest, ConvertedLegacyTest, TestCollectionAPI, TestNamedUpdateProcessors, TestMiniSolrCloudCluster, TestReplicaProperties, CoreAdminCreateDiscoverTest, TestConfigOverlay, ExitableDirectoryReaderTest, TestSerializedLuceneMatchVersion, RemoteQueryErrorTest, CursorMarkTest, TestRequestStatusCollectionAPI, TestSolrConfigHandlerCloud, TestSimpleTrackingShardHandler, TestConfig, TestSortingResponseWriter, TestDocumentBuilder, DistributedTermsComponentTest, HdfsDirectoryFactoryTest, CoreAdminRequestStatusTest, TestQueryUtils, TestSolrDeletionPolicy1, BasicZkTest, TestOverriddenPrefixQueryForCustomFieldType, TestLMJelinekMercerSimilarityFactory, TestBinaryResponseWriter, ZkControllerTest, OpenCloseCoreStressTest, HighlighterTest, ShowFileRequestHandlerTest, SolrIndexSplitterTest, SignatureUpdateProcessorFactoryTest, TestExtendedDismaxParser, SuggesterFSTTest, TestFoldingMultitermQuery, SpatialFilterTest, PolyFieldTest, SolrCoreCheckLockOnStartupTest, FieldMutatingUpdateProcessorTest, DirectUpdateHandlerOptimizeTest, DistanceFunctionTest, DebugComponentTest, DisMaxRequestHandlerTest, DirectSolrSpellCheckerTest, DocumentAnalysisRequestHandlerTest, TestOmitPositions, TermsComponentTest, DocumentBuilderTest, RequiredFieldsTest, FastVectorHighlighterTest, LoggingHandlerTest, IndexSchemaTest, ReturnFieldsTest, MBeansHandlerTest, UniqFieldsUpdateProcessorFactoryTest, JsonLoaderTest, TestPartialUpdateDeduplication, PingRequestHandlerTest, SearchHandlerTest, TestQuerySenderListener, TestSolrIndexConfig, CopyFieldTest, TestStressRecovery, TestSystemIdResolver, PrimUtilsTest, DateFieldTest, DOMUtilTest, RAMDirectoryFactoryTest, ClusterStateTest, TestLRUCache, TestUtils, ZkNodePropsTest, SliceStateTest, FileUtilsTest, CircularListTest, DistributedMLTComponentTest, CursorPagingTest, SolrTestCaseJ4Test, TestCursorMarkWithoutUniqueKey, TestDistributedMissingSort, TestTolerantSearch, TestJettySolrRunner, AssignTest, ConnectionManagerTest, CustomCollectionTest, DeleteReplicaTest, MultiThreadedOCPTest, ReplicationFactorTest]
   [junit4] Completed on J1 in 72.84s, 1 test, 1 error <<< FAILURES!

[...truncated 333 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 475 suites, 1892 tests, 3 errors, 58 ignored (9 assumptions)

Total time: 155 minutes 49 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 836978659 bytes
Compression is 0.0%
Took 2 min 29 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 793 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/793/

6 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
There were too many update fails - we expect it can happen, but shouldn't easily

Stack Trace:
java.lang.AssertionError: There were too many update fails - we expect it can happen, but shouldn't easily
	at __randomizedtesting.SeedInfo.seed([87FFF320CD4FE006:FABCCFA63B38DFE]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertFalse(Assert.java:68)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:222)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
Invalid content type: 

Stack Trace:
org.apache.http.ParseException: Invalid content type: 
	at __randomizedtesting.SeedInfo.seed([87FFF320CD4FE006:FABCCFA63B38DFE]:0)
	at org.apache.http.entity.ContentType.parse(ContentType.java:233)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:537)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:224)
	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:166)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:671)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test(FullSolrCloudDistribCmdsTest.java:151)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.handler.component.DistributedFacetPivotSmallTest.test

Error Message:
Captured an uncaught exception in thread: Thread[id=226, name=Thread-67, state=RUNNABLE, group=TGRP-DistributedFacetPivotSmallTest]

Stack Trace:
com.carrotsearch.randomizedtesting.UncaughtExceptionError: Captured an uncaught exception in thread: Thread[id=226, name=Thread-67, state=RUNNABLE, group=TGRP-DistributedFacetPivotSmallTest]
	at __randomizedtesting.SeedInfo.seed([87FFF320CD4FE006:FABCCFA63B38DFE]:0)
Caused by: java.lang.RuntimeException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:49545//collection1
	at __randomizedtesting.SeedInfo.seed([87FFF320CD4FE006]:0)
	at org.apache.solr.BaseDistributedSearchTestCase$5.run(BaseDistributedSearchTestCase.java:624)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:49545//collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:594)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:943)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$5.run(BaseDistributedSearchTestCase.java:619)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 6 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:35785/ndm/ry/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:35785/ndm/ry/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([87FFF320CD4FE006:FABCCFA63B38DFE]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28169/wj/y/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28169/wj/y/collection1
	at __randomizedtesting.SeedInfo.seed([87FFF320CD4FE006:FABCCFA63B38DFE]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28169/wj/y/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:47366/ytq/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:47366/ytq/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([87FFF320CD4FE006:FABCCFA63B38DFE]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:47366/ytq/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 53 more




Build Log:
[...truncated 9994 lines...]
   [junit4] Suite: org.apache.solr.handler.component.DistributedFacetPivotSmallTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/init-core-data-001
   [junit4]   2> 83333 T140 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 83334 T140 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 85156 T140 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/control/cores/collection1
   [junit4]   2> 86314 T140 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 86807 T140 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:38909
   [junit4]   2> 86808 T140 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/control/cores, hostContext=/, hostPort=38909}
   [junit4]   2> 86975 T140 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 86976 T140 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/control/'
   [junit4]   2> 87030 T140 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/control/solr.xml
   [junit4]   2> 87101 T140 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/control/cores
   [junit4]   2> 87102 T140 oasc.CoreContainer.<init> New CoreContainer 1351866607
   [junit4]   2> 87102 T140 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/control/]
   [junit4]   2> 87103 T140 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/control/lib
   [junit4]   2> 87104 T140 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/control/lib).
   [junit4]   2> 87220 T140 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 87221 T140 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 87221 T140 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 87222 T140 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 87222 T140 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 87223 T140 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 87223 T140 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 87223 T140 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 87224 T140 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 87224 T140 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 87224 T140 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 87225 T140 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 87226 T140 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 87226 T140 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 87226 T140 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 87228 T140 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/control/cores
   [junit4]   2> 87233 T140 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/control/cores/collection1, transient=false, dataDir=data/, name=collection1, loadOnStartup=true, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/control/cores/collection1/, configSet=collection1, schema=schema.xml}
   [junit4]   2> 87233 T140 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/control/cores/collection1/
   [junit4]   2> 87234 T140 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 87235 T152 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 87237 T152 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 87238 T152 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 87238 T152 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 87372 T152 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 87412 T152 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 87467 T152 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 87468 T152 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 87484 T152 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 87987 T152 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 88002 T152 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 88004 T152 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 88021 T152 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 88025 T152 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 88028 T152 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 88029 T152 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 88029 T152 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 88029 T152 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 88029 T152 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 88030 T152 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 88030 T152 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from configset /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1
   [junit4]   2> 88030 T152 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 88031 T152 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/control/cores/collection1/data/
   [junit4]   2> 88031 T152 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@23156b71
   [junit4]   2> 88038 T152 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/control/cores/collection1/data
   [junit4]   2> 88039 T152 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/control/cores/collection1/data/index/
   [junit4]   2> 88039 T152 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/control/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 88040 T152 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/control/cores/collection1/data/index
   [junit4]   2> 88040 T152 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=33, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=21.1015625, floorSegmentMB=1.1337890625, forceMergeDeletesPctAllowed=24.36751248358017, segmentsPerTier=42.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8964166397413699
   [junit4]   2> 88041 T152 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6f67d719 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@a5ae373),segFN=segments_1,generation=1}
   [junit4]   2> 88041 T152 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 88049 T152 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 88049 T152 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 88050 T152 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 88050 T152 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 88050 T152 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 88050 T152 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 88050 T152 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 88051 T152 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 88051 T152 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 88051 T152 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 88052 T152 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 88052 T152 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 88069 T152 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 88072 T152 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 88073 T152 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 88075 T152 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 88082 T152 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 88086 T152 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/csv,/get,/config,/admin/threads,tvrh,/mlt,spellCheckCompRH_Direct,spellCheckWithWordbreak,/admin/mbeans,spellCheckCompRH,spellCheckCompRH1,defaults,/update/json/docs,lazy,/admin/plugins,dismax,/admin/segments,/terms,/admin/luke,/update/json,/admin/logging,/search-facet-invariants,/update,/debug/dump,/schema,mock,/search-facet-def,/admin/ping,/admin/properties,/replication,spellCheckWithWordbreak_Direct,/admin/system,mltrh,standard,/admin/file
   [junit4]   2> 88088 T152 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 88089 T152 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 88090 T152 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 88091 T152 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 88092 T152 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 88092 T152 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 88093 T152 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@3091fa73
   [junit4]   2> 88094 T152 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6f67d719 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@a5ae373),segFN=segments_1,generation=1}
   [junit4]   2> 88094 T152 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 88094 T152 oass.SolrIndexSearcher.<init> Opening Searcher@7465364[collection1] main
   [junit4]   2> 88095 T152 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 88095 T152 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 88095 T152 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 88095 T152 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 88115 T152 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 88163 T152 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 88163 T153 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 88164 T153 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 88164 T153 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 88165 T153 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 88165 T152 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 88165 T153 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 88166 T140 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1
   [junit4]   2> 88166 T140 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 88166 T153 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 88167 T153 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 88168 T153 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 88169 T153 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 88170 T153 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7465364[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 89697 T140 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard0/cores/collection1
   [junit4]   2> 89699 T140 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 89702 T140 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:48693
   [junit4]   2> 89702 T140 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, hostPort=48693, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard0/cores}
   [junit4]   2> 89702 T140 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 89703 T140 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard0/'
   [junit4]   2> 89740 T140 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard0/solr.xml
   [junit4]   2> 89786 T140 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard0/cores
   [junit4]   2> 89786 T140 oasc.CoreContainer.<init> New CoreContainer 714349646
   [junit4]   2> 89787 T140 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard0/]
   [junit4]   2> 89787 T140 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard0/lib
   [junit4]   2> 89788 T140 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard0/lib).
   [junit4]   2> 89797 T140 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 89798 T140 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 89798 T140 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 89798 T140 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 89798 T140 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 89799 T140 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 89799 T140 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 89799 T140 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 89799 T140 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 89800 T140 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 89800 T140 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 89800 T140 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 89801 T140 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 89801 T140 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 89801 T140 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 89802 T140 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard0/cores
   [junit4]   2> 89804 T140 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {name=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard0/cores/collection1, loadOnStartup=true, dataDir=data/, config=solrconfig.xml, configSet=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard0/cores/collection1/, schema=schema.xml, transient=false}
   [junit4]   2> 89804 T140 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard0/cores/collection1/
   [junit4]   2> 89805 T140 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 89806 T164 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 89808 T164 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 89808 T164 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 89809 T164 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 89864 T164 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 89890 T164 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 89951 T164 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 89952 T164 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 89964 T164 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 90314 T164 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 90331 T164 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 90334 T164 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 90350 T164 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 90354 T164 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 90357 T164 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 90358 T164 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 90358 T164 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 90359 T164 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 90359 T164 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 90359 T164 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 90360 T164 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from configset /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1
   [junit4]   2> 90360 T164 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 90360 T164 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard0/cores/collection1/data/
   [junit4]   2> 90360 T164 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@23156b71
   [junit4]   2> 90363 T164 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard0/cores/collection1/data
   [junit4]   2> 90364 T164 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard0/cores/collection1/data/index/
   [junit4]   2> 90364 T164 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard0/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 90364 T164 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard0/cores/collection1/data/index
   [junit4]   2> 90365 T164 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=33, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=21.1015625, floorSegmentMB=1.1337890625, forceMergeDeletesPctAllowed=24.36751248358017, segmentsPerTier=42.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8964166397413699
   [junit4]   2> 90365 T164 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@73adc939 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1675f884),segFN=segments_1,generation=1}
   [junit4]   2> 90366 T164 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 90374 T164 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 90374 T164 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 90374 T164 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 90375 T164 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 90375 T164 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 90375 T164 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 90375 T164 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 90375 T164 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 90376 T164 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 90376 T164 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 90376 T164 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 90376 T164 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 90394 T164 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 90396 T164 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 90398 T164 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 90400 T164 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 90420 T164 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 90425 T164 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/csv,/get,/config,/admin/threads,tvrh,/mlt,spellCheckCompRH_Direct,spellCheckWithWordbreak,/admin/mbeans,spellCheckCompRH,spellCheckCompRH1,defaults,/update/json/docs,lazy,/admin/plugins,dismax,/admin/segments,/terms,/admin/luke,/update/json,/admin/logging,/search-facet-invariants,/update,/debug/dump,/schema,mock,/search-facet-def,/admin/ping,/admin/properties,/replication,spellCheckWithWordbreak_Direct,/admin/system,mltrh,standard,/admin/file
   [junit4]   2> 90426 T164 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 90427 T164 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 90429 T164 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 90429 T164 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 90430 T164 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 90430 T164 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 90431 T164 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@116e6c19
   [junit4]   2> 90432 T164 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@73adc939 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@1675f884),segFN=segments_1,generation=1}
   [junit4]   2> 90432 T164 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 90432 T164 oass.SolrIndexSearcher.<init> Opening Searcher@28da98e2[collection1] main
   [junit4]   2> 90433 T164 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 90433 T164 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 90433 T164 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 90433 T164 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 90457 T164 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 90507 T164 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 90508 T165 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 90509 T165 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 90509 T165 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 90509 T165 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 90510 T165 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 90511 T165 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 90511 T164 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 90512 T165 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 90512 T140 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1
   [junit4]   2> 90512 T140 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 90513 T165 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 90514 T165 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 90516 T165 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@28da98e2[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 92154 T140 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard1/cores/collection1
   [junit4]   2> 92157 T140 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 92160 T140 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:49545
   [junit4]   2> 92161 T140 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard1/cores, hostPort=49545}
   [junit4]   2> 92161 T140 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 92162 T140 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard1/'
   [junit4]   2> 92209 T140 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard1/solr.xml
   [junit4]   2> 92264 T140 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard1/cores
   [junit4]   2> 92265 T140 oasc.CoreContainer.<init> New CoreContainer 2000628010
   [junit4]   2> 92265 T140 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard1/]
   [junit4]   2> 92266 T140 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard1/lib
   [junit4]   2> 92267 T140 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard1/lib).
   [junit4]   2> 92280 T140 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 92281 T140 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 92281 T140 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 92282 T140 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 92282 T140 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 92282 T140 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 92283 T140 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 92283 T140 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 92283 T140 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 92284 T140 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 92284 T140 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 92285 T140 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 92285 T140 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 92286 T140 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 92286 T140 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 92287 T140 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard1/cores
   [junit4]   2> 92290 T140 oasc.CoreDescriptor.<init> CORE DESCRIPTOR: {schema=schema.xml, config=solrconfig.xml, dataDir=data/, loadOnStartup=true, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard1/cores/collection1/, name=collection1, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard1/cores/collection1, configSet=collection1}
   [junit4]   2> 92290 T140 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard1/cores/collection1/
   [junit4]   2> 92291 T140 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 92292 T176 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 92293 T176 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 92294 T176 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 92295 T176 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 92349 T176 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 92383 T176 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 92461 T176 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 92462 T176 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 92476 T176 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 92956 T176 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 92979 T176 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 92983 T176 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 93007 T176 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 93013 T176 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 93018 T176 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 93019 T176 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 93019 T176 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 93019 T176 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 93020 T176 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 93020 T176 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 93021 T176 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from configset /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1
   [junit4]   2> 93021 T176 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 93021 T176 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard1/cores/collection1/data/
   [junit4]   2> 93022 T176 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@23156b71
   [junit4]   2> 93025 T176 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard1/cores/collection1/data
   [junit4]   2> 93027 T176 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard1/cores/collection1/data/index/
   [junit4]   2> 93027 T176 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard1/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 93028 T176 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.component.DistributedFacetPivotSmallTest 87FFF320CD4FE006-001/tempDir-001/shard1/cores/collection1/data/index
   [junit4]   2> 93028 T176 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=33, maxMergeAtOnceExplicit=13, maxMergedSegmentMB=21.1015625, floorSegmentMB=1.1337890625, forceMergeDeletesPctAllowed=24.36751248358017, segmentsPerTier=42.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.8964166397413699
   [junit4]   2> 93029 T176 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@19335100 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@780ae3ec),segFN=segments_1,generation=1}
   [junit4]   2> 93030 T176 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 93041 T176 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 93042 T176 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 93042 T176 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 93043 T176 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 93043 T176 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 93043 T176 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 93043 T176 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 93044 T176 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 93044 T176 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 93045 T176 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 93045 T176 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 93045 T176 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 93073 T176 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 93076 T176 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 93079 T176 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 93082 T176 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 93095 T176 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 93102 T176 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update/csv,/get,/config,/admin/threads,tvrh,/mlt,spellCheckCompRH_Direct,spellCheckWithWordbreak,/admin/mbeans,spellCheckCompRH,spellCheckCompRH1,defaults,/update/json/docs,lazy,/admin/plugins,dismax,/admin/segments,/terms,/admin/luke,/update/json,/admin/logging,/search-facet-invariants,/update,/debug/dump,/schema,mock,/search-facet-def,/admin/ping,/admin/properties,/replication,spellCheckWithWordbreak_Direct,/admin/system,mltrh,standard,/admin/file
   [junit4]   2> 93104 T176 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 93106 T176 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 93108 T176 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 93108 T176 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 93110 T176 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 93111 T176 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 93112 T176 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@569ecc6d
   [junit4]   2> 93113 T176 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@19335100 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@780ae3ec),segFN=segments_1,generation=1}
   [junit4]   2> 93113 T176 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 93114 T176 oass.SolrIndexSearcher.<init> Opening Searcher@43c8bbed[collection1] main
   [junit4]   2> 93114 T176 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 93115 T176 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 93115 T176 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 93115 T176 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 93135 T176 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 93201 T176 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 93202 T177 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 93203 T177 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 93204 T177 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 93204 T177 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 93205 T177 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 93205 T176 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 93206 T140 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1
   [junit4]   2> 93206 T177 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 93206 T140 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 93207 T177 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 93208 T177 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell i

[...truncated too long message...]

ollection-queue-work
   [junit4]   2> 		2	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 2250826 T39045 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:28176/wj/y
   [junit4]   2> 2250826 T39045 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:28169/wj/y
   [junit4]   2> 2250827 T39045 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:28161/wj/y
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=87FFF320CD4FE006 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=es_MX -Dtests.timezone=Indian/Maldives -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   63.6s J2 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28169/wj/y/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([87FFF320CD4FE006:FABCCFA63B38DFE]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:28169/wj/y/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	... 54 more
   [junit4]   2> 2250835 T39045 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 87FFF320CD4FE006-001
   [junit4]   2> 63642 T39044 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {multiDefault=PostingsFormat(name=Memory doPackFST= false), text=PostingsFormat(name=LuceneVarGapDocFreqInterval), intDefault=PostingsFormat(name=Memory doPackFST= true), a_t=PostingsFormat(name=Memory doPackFST= true), range_facet_l=PostingsFormat(name=Memory doPackFST= true), _version_=PostingsFormat(name=Memory doPackFST= true), id=BlockTreeOrds(blocksize=128), timestamp=PostingsFormat(name=Memory doPackFST= true)}, docValues:{timestamp=DocValuesFormat(name=Asserting)}, sim=RandomSimilarityProvider(queryNorm=true,coord=no): {}, locale=es_MX, timezone=Indian/Maldives
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=164337960,total=317718528
   [junit4]   2> NOTE: All tests run in this JVM: [TestRTGBase, DateFieldTest, TestRealTimeGet, SuggestComponentTest, LeaderElectionIntegrationTest, AddSchemaFieldsUpdateProcessorFactoryTest, SolrPluginUtilsTest, DeleteLastCustomShardedReplicaTest, HdfsChaosMonkeySafeLeaderTest, TestReloadAndDeleteDocs, DeleteReplicaTest, DocumentBuilderTest, TestFileDictionaryLookup, SolrIndexSplitterTest, HighlighterMaxOffsetTest, QueryEqualityTest, HardAutoCommitTest, TestManagedSchemaFieldResource, TestReload, BasicDistributedZk2Test, TestShortCircuitedRequests, TestBinaryField, TestSuggestSpellingConverter, PeerSyncTest, CoreMergeIndexesAdminHandlerTest, TestMergePolicyConfig, TestManagedSynonymFilterFactory, SpatialHeatmapFacetsTest, HdfsDirectoryTest, TestFieldSortValues, TestFoldingMultitermQuery, InfoHandlerTest, DateRangeFieldTest, TestObjectReleaseTracker, ChaosMonkeyNothingIsSafeTest, NotRequiredUniqueKeyTest, DistributedQueueTest, AnalyticsQueryTest, SolrRequestParserTest, TestPerFieldSimilarity, TestReplicationHandlerBackup, SynonymTokenizerTest, TestReqParamsAPI, TestFieldCollectionResource, ReturnFieldsTest, DirectUpdateHandlerTest, TestCursorMarkWithoutUniqueKey, TestTrackingShardHandlerFactory, TestFreeTextSuggestions, BadCopyFieldTest, TestAnalyzedSuggestions, RollingRestartTest, CachingDirectoryFactoryTest, TestCSVResponseWriter, TestManagedSchemaDynamicFieldResource, TestNonDefinedSimilarityFactory, DocValuesMissingTest, TestQuerySenderNoQuery, ShardRoutingTest, TestRandomFaceting, ZkSolrClientTest, ZkCLITest, ShardRoutingCustomTest, TestDistributedSearch, LeaderInitiatedRecoveryOnCommitTest, MigrateRouteKeyTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerRolesTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J2 in 64.43s, 1 test, 1 error <<< FAILURES!

[...truncated 320 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 475 suites, 1892 tests, 5 errors, 1 failure, 58 ignored (9 assumptions)

Total time: 170 minutes 11 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 779406755 bytes
Compression is 0.0%
Took 2 min 26 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 792 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/792/

4 tests failed.
REGRESSION:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:29770/c8n_1x3_lf_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:29770/c8n_1x3_lf_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([2281FD7A30CC475D:AAD5C2A09E302AA5]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.sendDocsWithRetry(AbstractFullDistribZkTestBase.java:790)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:483)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:172)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:29770/c8n_1x3_lf_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:594)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:22614/dtfx/q/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22614/dtfx/q/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([2281FD7A30CC475D:AAD5C2A09E302AA5]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:63417/d/ld/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:63417/d/ld/collection1
	at __randomizedtesting.SeedInfo.seed([2281FD7A30CC475D:AAD5C2A09E302AA5]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:63417/d/ld/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:63530/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:63530/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([2281FD7A30CC475D:AAD5C2A09E302AA5]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:63530/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 53 more




Build Log:
[...truncated 10504 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/init-core-data-001
   [junit4]   2> 1088932 T3243 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /dtfx/q
   [junit4]   2> 1088940 T3243 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1088940 T3244 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1089040 T3243 oasc.ZkTestServer.run start zk server on port:22601
   [junit4]   2> 1089041 T3243 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1089042 T3243 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1089051 T3251 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@790f05aa name:ZooKeeperConnection Watcher:127.0.0.1:22601 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1089052 T3243 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1089052 T3243 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1089052 T3243 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1089055 T3243 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1089056 T3243 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1089058 T3254 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4bb0e761 name:ZooKeeperConnection Watcher:127.0.0.1:22601/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1089058 T3243 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1089059 T3243 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1089059 T3243 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1089061 T3243 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1089062 T3243 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1089064 T3243 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1089066 T3243 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1089066 T3243 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1089069 T3243 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1089070 T3243 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1089173 T3243 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1089173 T3243 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1089175 T3243 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1089176 T3243 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1089178 T3243 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1089178 T3243 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1089180 T3243 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1089181 T3243 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1089182 T3243 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1089183 T3243 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1089184 T3243 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1089185 T3243 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1089187 T3243 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1089187 T3243 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1089192 T3243 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1089193 T3243 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1089195 T3243 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1089196 T3243 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1090121 T3243 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/control-001/cores/collection1
   [junit4]   2> 1090124 T3243 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1090128 T3243 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:22605
   [junit4]   2> 1090128 T3243 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/tempDir-001/control/data, hostPort=22604, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/control-001/cores, hostContext=/dtfx/q}
   [junit4]   2> 1090129 T3243 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 1090129 T3243 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/control-001/'
   [junit4]   2> 1090158 T3243 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/control-001/solr.xml
   [junit4]   2> 1090193 T3243 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/control-001/cores
   [junit4]   2> 1090193 T3243 oasc.CoreContainer.<init> New CoreContainer 1439253723
   [junit4]   2> 1090194 T3243 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/control-001/]
   [junit4]   2> 1090194 T3243 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/control-001/lib
   [junit4]   2> 1090194 T3243 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/control-001/lib).
   [junit4]   2> 1090203 T3243 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1090203 T3243 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1090204 T3243 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1090204 T3243 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1090204 T3243 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1090204 T3243 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1090204 T3243 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1090205 T3243 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1090205 T3243 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1090205 T3243 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1090205 T3243 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1090206 T3243 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1090206 T3243 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1090206 T3243 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1090207 T3243 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1090207 T3243 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:22601/solr
   [junit4]   2> 1090207 T3243 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1090207 T3243 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1090208 T3243 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1090210 T3266 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@28aa8275 name:ZooKeeperConnection Watcher:127.0.0.1:22601 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1090210 T3243 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1090211 T3243 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1090212 T3243 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1090213 T3269 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@22e52c17 name:ZooKeeperConnection Watcher:127.0.0.1:22601/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1090214 T3243 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1090215 T3243 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1090217 T3243 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1090219 T3243 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1090221 T3243 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1090223 T3243 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1090225 T3243 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1090227 T3243 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1090228 T3243 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1090229 T3243 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22604_dtfx%2Fq
   [junit4]   2> 1090230 T3243 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22604_dtfx%2Fq
   [junit4]   2> 1090231 T3243 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1090233 T3243 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1090234 T3243 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1090235 T3243 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:22604_dtfx%2Fq
   [junit4]   2> 1090236 T3243 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1090237 T3243 oasc.Overseer.start Overseer (id=93511133072916483-127.0.0.1:22604_dtfx%2Fq-n_0000000000) starting
   [junit4]   2> 1090239 T3243 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1090244 T3243 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1090245 T3271 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1090245 T3243 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1090245 T3270 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1090248 T3243 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/control-001/cores
   [junit4]   1> CORE DESCRIPTOR: {loadOnStartup=true, schema=schema.xml, coreNodeName=, name=collection1, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/control-001/cores/collection1/, transient=false, shard=, dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/control-001/cores/collection1, collection=control_collection}
   [junit4]   2> 1090250 T3243 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/control-001/cores/collection1/
   [junit4]   2> 1090250 T3243 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1090251 T3273 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1090251 T3273 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1090252 T3269 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1090252 T3273 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1090253 T3270 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:22604_dtfx%2Fq",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22604/dtfx/q",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 1090253 T3270 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:22604_dtfx%2Fq",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22604/dtfx/q",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 1090253 T3270 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1090254 T3270 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1090255 T3269 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1091252 T3273 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1091253 T3273 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1091253 T3273 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1091254 T3273 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1091254 T3273 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1091254 T3273 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/control-001/cores/collection1/'
   [junit4]   2> 1091266 T3273 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1091268 T3273 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1091276 T3273 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1091291 T3273 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 1091323 T3273 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1091425 T3273 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1091434 T3273 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1091694 T3273 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1091705 T3273 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1091707 T3273 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1091716 T3273 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1091719 T3273 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1091721 T3273 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1091722 T3273 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1091722 T3273 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1091722 T3273 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1091723 T3273 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1091723 T3273 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1091723 T3273 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1091724 T3273 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1091724 T3273 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/control-001/cores/collection1/data/
   [junit4]   2> 1091724 T3273 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2128d41b
   [junit4]   2> 1091725 T3273 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/control-001/cores/collection1/data
   [junit4]   2> 1091725 T3273 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1091725 T3273 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1091726 T3273 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/control-001/cores/collection1/data/index
   [junit4]   2> 1091726 T3273 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=4, maxMergeAtOnceExplicit=7, maxMergedSegmentMB=24.3515625, floorSegmentMB=2.111328125, forceMergeDeletesPctAllowed=6.956093903480815, segmentsPerTier=43.0, maxCFSSegmentSizeMB=1.4384765625, noCFSRatio=1.0
   [junit4]   2> 1091727 T3273 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@325f701a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@17b8d4cc),segFN=segments_1,generation=1}
   [junit4]   2> 1091727 T3273 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1091732 T3273 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1091732 T3273 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1091732 T3273 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1091732 T3273 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1091732 T3273 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1091733 T3273 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1091733 T3273 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1091733 T3273 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1091733 T3273 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1091745 T3273 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1091747 T3273 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1091749 T3273 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1091750 T3273 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1091755 T3273 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/mbeans,/admin/segments,/update/csv,/admin/threads,/schema,/update/json/docs,/admin/system,/admin/plugins,/replication,/config,standard,/admin/file,/admin/logging,/admin/luke,/get,/update/json,/admin/properties,/update,/admin/ping
   [junit4]   2> 1091756 T3273 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1091757 T3273 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1091757 T3273 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1091758 T3273 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1091758 T3273 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1091759 T3273 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=45, maxMergedSegmentMB=25.6787109375, floorSegmentMB=1.7685546875, forceMergeDeletesPctAllowed=6.563060040238514, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1091759 T3273 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@325f701a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@17b8d4cc),segFN=segments_1,generation=1}
   [junit4]   2> 1091760 T3273 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1091760 T3273 oass.SolrIndexSearcher.<init> Opening Searcher@1509b155[collection1] main
   [junit4]   2> 1091760 T3273 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1091761 T3273 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1091761 T3273 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1091762 T3273 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1091762 T3273 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1091762 T3273 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1091762 T3273 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1091763 T3273 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1091763 T3273 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1091763 T3273 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1091764 T3274 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1509b155[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1091764 T3273 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1091765 T3277 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:22604/dtfx/q collection:control_collection shard:shard1
   [junit4]   2> 1091765 T3243 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 1091765 T3243 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1091766 T3277 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1091766 T3243 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1091767 T3243 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1091768 T3281 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@46ffc119 name:ZooKeeperConnection Watcher:127.0.0.1:22601/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1091768 T3243 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1091768 T3243 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1091768 T3243 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1091770 T3277 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1091771 T3269 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1091771 T3243 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 1091771 T3277 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1091771 T3243 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 1091772 T3270 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1091772 T3277 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1091772 T3243 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> ASYNC  NEW_CORE C364 name=collection1 org.apache.solr.core.SolrCore@4f3b87f9 url=http://127.0.0.1:22604/dtfx/q/collection1 node=127.0.0.1:22604_dtfx%2Fq C364_STATE=coll:control_collection core:collection1 props:{core=collection1, node_name=127.0.0.1:22604_dtfx%2Fq, base_url=http://127.0.0.1:22604/dtfx/q, state=down}
   [junit4]   2> 1091772 T3277 C364 P22604 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:22604/dtfx/q/collection1/
   [junit4]   2> 1091773 T3243 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1091773 T3281 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1091773 T3269 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1091774 T3284 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@750ba046 name:ZooKeeperConnection Watcher:127.0.0.1:22601/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1091773 T3277 C364 P22604 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1091774 T3243 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1091774 T3277 C364 P22604 oasc.SyncStrategy.syncToMe http://127.0.0.1:22604/dtfx/q/collection1/ has no replicas
   [junit4]   2> 1091775 T3243 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1091775 T3277 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:22604/dtfx/q/collection1/ shard1
   [junit4]   2> 1091775 T3277 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1091776 T3269 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1091777 T3270 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 1091777 T3270 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1091780 T3270 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22604/dtfx/q",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1091780 T3270 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 1091781 T3269 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1091781 T3281 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1091783 T3269 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1091784 T3281 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1091784 T3269 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1091829 T3277 oasc.ZkController.register We are http://127.0.0.1:22604/dtfx/q/collection1/ and leader is http://127.0.0.1:22604/dtfx/q/collection1/
   [junit4]   2> 1091830 T3277 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:22604/dtfx/q
   [junit4]   2> 1091830 T3277 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1091830 T3277 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1091830 T3277 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1091832 T3269 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1091832 T3270 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:22604_dtfx%2Fq",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22604/dtfx/q",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null} current state version: 4
   [junit4]   2> 1091833 T3270 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:22604_dtfx%2Fq",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22604/dtfx/q",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 1091936 T3281 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1091936 T3269 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1092686 T3243 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/shard-1-001/cores/collection1
   [junit4]   2> 1092689 T3243 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/shard-1-001
   [junit4]   2> 1092689 T3243 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1092692 T3243 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:22611
   [junit4]   2> 1092692 T3243 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=22610, solrconfig=solrconfig.xml, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/tempDir-001/jetty1, hostContext=/dtfx/q, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/shard-1-001/cores}
   [junit4]   2> 1092693 T3243 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 1092693 T3243 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/shard-1-001/'
   [junit4]   2> 1092722 T3243 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/shard-1-001/solr.xml
   [junit4]   2> 1092757 T3243 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/shard-1-001/cores
   [junit4]   2> 1092758 T3243 oasc.CoreContainer.<init> New CoreContainer 1633642111
   [junit4]   2> 1092758 T3243 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/shard-1-001/]
   [junit4]   2> 1092758 T3243 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/shard-1-001/lib
   [junit4]   2> 1092759 T3243 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/shard-1-001/lib).
   [junit4]   2> 1092768 T3243 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1092768 T3243 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1092768 T3243 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1092769 T3243 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1092769 T3243 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1092769 T3243 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1092769 T3243 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1092770 T3243 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1092770 T3243 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1092770 T3243 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1092770 T3243 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1092771 T3243 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1092771 T3243 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1092771 T3243 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1092771 T3243 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1092772 T3243 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:22601/solr
   [junit4]   2> 1092772 T3243 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1092772 T3243 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1092773 T3243 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1092774 T3296 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@214c6fde name:ZooKeeperConnection Watcher:127.0.0.1:22601 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1092775 T3243 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1092775 T3243 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1092777 T3243 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1092778 T3299 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@73fc612c name:ZooKeeperConnection Watcher:127.0.0.1:22601/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1092778 T3243 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1092781 T3243 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1093786 T3243 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:22610_dtfx%2Fq
   [junit4]   2> 1093786 T3243 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:22610_dtfx%2Fq
   [junit4]   2> 1093788 T3243 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1093790 T3243 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/shard-1-001/cores
   [junit4]   1> CORE DESCRIPTOR: {dataDir=data/, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/shard-1-001/cores/collection1, collection=collection1, shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/shard-1-001/cores/collection1/, name=collection1, schema=schema.xml, config=solrconfig.xml, loadOnStartup=true, transient=false}
   [junit4]   2> 1093792 T3243 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/shard-1-001/cores/collection1/
   [junit4]   2> 1093793 T3243 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1093794 T3300 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1093795 T3300 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1093796 T3269 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1093796 T3300 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 1093796 T3300 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 1093796 T3270 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:22610_dtfx%2Fq",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22610/dtfx/q",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null} current state version: 5
   [junit4]   2> 1093797 T3300 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 1093797 T3270 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:22610_dtfx%2Fq",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:22610/dtfx/q",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null}
   [junit4]   2> 1093797 T3270 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 1093797 T3270 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1093798 T3300 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 1093798 T3300 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1093798 T3270 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 1093799 T3299 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1093799 T3299 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 1094798 T3300 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1094799 T3300 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1094799 T3300 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1094799 T3300 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1094800 T3300 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1094800 T3300 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1094812 T3300 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1094814 T3300 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1094822 T3300 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1094837 T3300 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 1094896 T3300 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1094999 T3300 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1095012 T3300 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1095371 T3300 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1095382 T3300 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1095384 T3300 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1095393 T3300 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1095396 T3300 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1095399 T3300 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1095399 T3300 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1095400 T3300 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1095400 T3300 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1095400 T3300 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1095400 T3300 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1095401 T3300 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1095401 T3300 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1095401 T3300 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1095401 T3300 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2128d41b
   [junit4]   2> 1095402 T3300 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1095402 T3300 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1095403 T3300 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1095403 T3300 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest 2281FD7A30CC475D-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1095403 T3300 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=4, maxMergeAtOnceExplicit=7, maxMergedSegmentMB=24.3515625, floorSegmentMB=2.111328125, forceMergeDeletesPctAllowed=6.956093903480815, segmentsPerTier=43.0, maxCFSSegmentSizeMB=1.4384765625, noCFSRatio=1.0
   [junit4]   2> 1095404 T3300 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@64823a1 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6a1b223f),segFN=segments_1,generation=1}
   [junit4]   2> 1095404 T3300 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1095409 T3300 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1095409 T3300 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1095409 T3300 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1095409 T3300 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1095409 T3300 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1095410 T3300 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1095410 T3300 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1095410 T3300 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1095410 T3300 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1095422 T3300 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1095424 T3300 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1095425 T3300 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1095427 T3300 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1095432 T3300 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/mbeans,/admin/segments,/update/csv,/admin/threads,/schema,/update/json/docs,/admin/system,/admin/plugins,/replication,/config,standard,/admin/file,/admin/logging,/admin/luke,/get,/update/json,/admin/properties,/update,/admin/ping
   [junit4]   2> 1095434 T3300 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1095435 T3300 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1095435 T3300 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1095436 T3300 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1095436 T3300 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1095437 T3300 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=14, maxMergeAtOnceExplicit=45, maxMergedSegmentMB=25.6787109375, floorSegmentMB=1.7685546875, forceMergeDeletesPctAllowed=6.563060040238514, segmentsPerTier=13.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 1095437 T3300 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@64823a1 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@6a1b223f),segFN=segments_1,generation=1}
   [junit4]   2> 1095438 T3300 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1095438 T3300 oass.SolrIndexSearcher.<init> Opening Searcher@dfed5c6[collection1] main
   [junit4]   2> 1095438 T3300 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1095439 T3300 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1095439 T3300 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1095439 T3300 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1095440 T3300 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1095440 T3300 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1095440 T3300 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1095440 T3300 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1095441 T3300 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1095441 T3300 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1095442 T3301 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@dfed5c6[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1095442 T3300 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1095443 T3304 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:22610/dtfx/q collection:collection1 shard:shard1
   [junit4]   2> 1095443 T3243 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J

[...truncated too long message...]

.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderFailoverAfterPartitionTest -Dtests.method=test -Dtests.seed=2281FD7A30CC475D -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=es_ES -Dtests.timezone=GB -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR    988s J2 | LeaderFailoverAfterPartitionTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:29770/c8n_1x3_lf_shard1_replica3
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([2281FD7A30CC475D:AAD5C2A09E302AA5]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.sendDocsWithRetry(AbstractFullDistribZkTestBase.java:790)
   [junit4]    > 	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:483)
   [junit4]    > 	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:172)
   [junit4]    > 	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:29770/c8n_1x3_lf_shard1_replica3
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:594)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: java.net.SocketTimeoutException: Read timed out
   [junit4]    > 	at java.net.SocketInputStream.socketRead0(Native Method)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:152)
   [junit4]    > 	at java.net.SocketInputStream.read(SocketInputStream.java:122)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
   [junit4]    > 	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
   [junit4]    > 	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	... 54 more
   [junit4]   2> 3114447 T41450 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest 2281FD7A30CC475D-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {id=Lucene50(blocksize=128), range_facet_l=BlockTreeOrds(blocksize=128), timestamp=BlockTreeOrds(blocksize=128), text=FSTOrd50, a_t=BlockTreeOrds(blocksize=128), multiDefault=BlockTreeOrds(blocksize=128), _version_=Lucene50(blocksize=128), intDefault=Lucene50(blocksize=128)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=es_ES, timezone=GB
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=18827184,total=362283008
   [junit4]   2> NOTE: All tests run in this JVM: [TestSolrQueryParser, BadComponentTest, SynonymTokenizerTest, MoreLikeThisHandlerTest, TestRemoteStreaming, DocValuesMissingTest, TestFieldSortValues, InfoHandlerTest, SpatialHeatmapFacetsTest, SolrIndexSplitterTest, DocumentBuilderTest, DistributedIntervalFacetingTest, TestExtendedDismaxParser, TestReRankQParserPlugin, TestExceedMaxTermLength, FileBasedSpellCheckerTest, BasicDistributedZk2Test, TestMergePolicyConfig, TestCSVResponseWriter, TestFieldTypeResource, TestReloadAndDeleteDocs, SimpleMLTQParserTest, TestManagedSchemaDynamicFieldResource, DistribJoinFromCollectionTest, HardAutoCommitTest, SuggestComponentTest, SpatialFilterTest, DirectUpdateHandlerOptimizeTest, SliceStateTest, TestComplexPhraseQParserPlugin, TestSuggestSpellingConverter, TestManagedSynonymFilterFactory, QueryEqualityTest, ReturnFieldsTest, TestFoldingMultitermQuery, TestReplicationHandlerBackup, TestManagedResource, TestNonDefinedSimilarityFactory, TestLazyCores, PeerSyncTest, TestAnalyzedSuggestions, HdfsSyncSliceTest, TestObjectReleaseTracker, BadCopyFieldTest, CurrencyFieldOpenExchangeTest, TestRawResponseWriter, TestLeaderElectionZkExpiry, SegmentsInfoRequestHandlerTest, RAMDirectoryFactoryTest, SolrCoreCheckLockOnStartupTest, SaslZkACLProviderTest, HdfsCollectionsAPIDistributedZkTest, TestRebalanceLeaders, DateFieldTest, TestHashPartitioner, LeaderElectionIntegrationTest, TestBulkSchemaConcurrent, UnloadDistributedZkTest, SyncSliceTest, LeaderElectionTest, ZkSolrClientTest, ZkCLITest, ShardRoutingCustomTest, TestDistributedSearch, TestSystemIdResolver, PrimUtilsTest, TestLRUCache, ZkNodePropsTest, CircularListTest, TestRTGBase, CursorPagingTest, TestTolerantSearch, TestJettySolrRunner, AliasIntegrationTest, CollectionsAPIAsyncDistributedZkTest, ConnectionManagerTest, DeleteLastCustomShardedReplicaTest, DistribCursorPagingTest, LeaderFailoverAfterPartitionTest]
   [junit4] Completed on J2 in 987.99s, 1 test, 1 error <<< FAILURES!

[...truncated 15 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 474 suites, 1890 tests, 4 errors, 58 ignored (9 assumptions)

Total time: 166 minutes 49 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 807453134 bytes
Compression is 0.0%
Took 2 min 35 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 791 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/791/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:61960/g/oc/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:61960/g/oc/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([18C2A3D48792D947:90969C0E296EB4BF]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:64353/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:64353/collection1
	at __randomizedtesting.SeedInfo.seed([18C2A3D48792D947:90969C0E296EB4BF]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:64353/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48159/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48159/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([18C2A3D48792D947:90969C0E296EB4BF]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48159/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 53 more




Build Log:
[...truncated 9995 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/init-core-data-001
   [junit4]   2> 43078 T39 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 43140 T39 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 43153 T40 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 43556 T39 oasc.ZkTestServer.run start zk server on port:56160
   [junit4]   2> 43778 T39 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 44127 T39 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 44358 T47 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@66108c7d name:ZooKeeperConnection Watcher:127.0.0.1:56160 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 44359 T39 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 44370 T39 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 44390 T39 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 44562 T39 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 44564 T39 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 44567 T50 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4de43413 name:ZooKeeperConnection Watcher:127.0.0.1:56160/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 44567 T39 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 44568 T39 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 44599 T39 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 44605 T39 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 44610 T39 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 44613 T39 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 44618 T39 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 44678 T39 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 44686 T39 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 44687 T39 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 44692 T39 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 44693 T39 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 44698 T39 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 44699 T39 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 44704 T39 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 44705 T39 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 44709 T39 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 44710 T39 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 44715 T39 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 44716 T39 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 44722 T39 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 44722 T39 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 44728 T39 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 44729 T39 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 44734 T39 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 44735 T39 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 44740 T39 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 44741 T39 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 46399 T39 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/control-001/cores/collection1
   [junit4]   2> 47136 T39 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 47557 T39 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:24212
   [junit4]   2> 47558 T39 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=48159, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/control-001/cores, hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/tempDir-001/control/data}
   [junit4]   2> 47833 T39 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 47833 T39 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/control-001/'
   [junit4]   2> 47920 T39 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/control-001/solr.xml
   [junit4]   2> 48075 T39 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/control-001/cores
   [junit4]   2> 48077 T39 oasc.CoreContainer.<init> New CoreContainer 252280818
   [junit4]   2> 48077 T39 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/control-001/]
   [junit4]   2> 48078 T39 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/control-001/lib
   [junit4]   2> 48080 T39 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/control-001/lib).
   [junit4]   2> 48380 T39 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 48380 T39 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 48381 T39 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 48381 T39 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 48382 T39 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 48382 T39 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 48383 T39 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 48383 T39 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 48384 T39 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 48384 T39 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 48385 T39 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 48386 T39 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 48387 T39 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 48387 T39 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 48388 T39 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 48389 T39 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:56160/solr
   [junit4]   2> 48487 T39 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 48488 T39 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 48490 T39 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 48494 T63 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@76033f20 name:ZooKeeperConnection Watcher:127.0.0.1:56160 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 48494 T39 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 48495 T39 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 48527 T39 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 48530 T66 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@e0944f1 name:ZooKeeperConnection Watcher:127.0.0.1:56160/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 48530 T39 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 48550 T39 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 48569 T39 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 48579 T39 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 48584 T39 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 48589 T39 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 48625 T39 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 48630 T39 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 48634 T39 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 48637 T39 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48159_
   [junit4]   2> 48645 T39 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:48159_
   [junit4]   2> 48649 T39 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 48652 T39 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 48656 T39 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 48685 T39 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:48159_
   [junit4]   2> 48686 T39 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 48691 T39 oasc.Overseer.start Overseer (id=93505368012947459-127.0.0.1:48159_-n_0000000000) starting
   [junit4]   2> 48716 T39 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 48767 T39 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 48800 T68 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 48800 T39 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 48843 T67 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 48849 T39 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/control-001/cores
   [junit4]   1> CORE DESCRIPTOR: {loadOnStartup=true, dataDir=data/, coreNodeName=, transient=false, shard=, config=solrconfig.xml, name=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/control-001/cores/collection1, collection=control_collection, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/control-001/cores/collection1/, schema=schema.xml}
   [junit4]   2> 48873 T39 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/control-001/cores/collection1/
   [junit4]   2> 48874 T39 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 48876 T70 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 48877 T70 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 48898 T66 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 48899 T70 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 48910 T67 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48159",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:48159_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 48985 T67 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48159",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:48159_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 48994 T67 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 49051 T67 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 49075 T66 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 49901 T70 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 49901 T70 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 49903 T70 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 49903 T70 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 49905 T70 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 49906 T70 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/control-001/cores/collection1/'
   [junit4]   2> 49985 T70 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 50017 T70 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 50063 T70 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 50099 T70 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 50163 T70 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 50165 T70 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 50213 T70 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 50587 T70 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 50604 T70 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 50606 T70 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 50633 T70 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 50638 T70 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 50643 T70 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 50644 T70 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 50645 T70 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 50645 T70 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 50646 T70 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 50647 T70 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 50647 T70 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 50647 T70 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 50648 T70 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/control-001/cores/collection1/data/
   [junit4]   2> 50653 T70 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@56f3b98c
   [junit4]   2> 50664 T70 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/control-001/cores/collection1/data
   [junit4]   2> 50664 T70 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/control-001/cores/collection1/data/index/
   [junit4]   2> 50664 T70 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 50665 T70 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/control-001/cores/collection1/data/index
   [junit4]   2> 50665 T70 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=71.2314453125, floorSegmentMB=1.5166015625, forceMergeDeletesPctAllowed=23.887340780558468, segmentsPerTier=50.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7575247963883267
   [junit4]   2> 50666 T70 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@6de7af6a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ad9085),segFN=segments_1,generation=1}
   [junit4]   2> 50666 T70 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 50675 T70 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 50723 T70 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 50723 T70 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 50723 T70 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 50724 T70 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 50724 T70 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 50725 T70 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 50726 T70 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 50726 T70 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 50739 T70 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 50746 T70 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 50748 T70 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 50751 T70 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 50762 T70 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/ping,/admin/plugins,/admin/luke,/admin/properties,/admin/threads,/update/json/docs,/schema,/admin/logging,/config,/admin/system,/admin/segments,/admin/file,/update/csv,/get,/update/json,/admin/mbeans,standard,/replication,/update
   [junit4]   2> 50763 T70 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 50777 T70 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 50778 T70 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 50789 T70 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 50789 T70 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 50790 T70 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=35, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=0.78515625, noCFSRatio=1.0]
   [junit4]   2> 50791 T70 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@6de7af6a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5ad9085),segFN=segments_1,generation=1}
   [junit4]   2> 50791 T70 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 50792 T70 oass.SolrIndexSearcher.<init> Opening Searcher@ee45211[collection1] main
   [junit4]   2> 50792 T70 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 50794 T70 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 50794 T70 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 50802 T70 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 50802 T70 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 50802 T70 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 50803 T70 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 50804 T70 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 50804 T70 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 50804 T70 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 50807 T71 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@ee45211[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 50811 T70 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 50813 T74 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:48159 collection:control_collection shard:shard1
   [junit4]   2> 50813 T39 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 50814 T39 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 50838 T74 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 50847 T74 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 50850 T66 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 50851 T74 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 50851 T74 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 50852 T67 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C9 name=collection1 org.apache.solr.core.SolrCore@3e134a3f url=http://127.0.0.1:48159/collection1 node=127.0.0.1:48159_ C9_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=http://127.0.0.1:48159, node_name=127.0.0.1:48159_, state=down}
   [junit4]   2> 50858 T74 C9 P48159 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:48159/collection1/
   [junit4]   2> 50858 T74 C9 P48159 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 50859 T74 C9 P48159 oasc.SyncStrategy.syncToMe http://127.0.0.1:48159/collection1/ has no replicas
   [junit4]   2> 50859 T74 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:48159/collection1/ shard1
   [junit4]   2> 50864 T66 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 50871 T74 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 50877 T66 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 50879 T67 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48159",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 50899 T39 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 50899 T39 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 50901 T78 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@77fb258f name:ZooKeeperConnection Watcher:127.0.0.1:56160/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 50902 T39 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 50902 T39 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 50903 T39 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 50909 T39 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 50909 T39 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 50909 T39 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 50910 T39 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 50912 T81 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4a389db3 name:ZooKeeperConnection Watcher:127.0.0.1:56160/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 50912 T39 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 50913 T39 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 50916 T66 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 50918 T67 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 50919 T67 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 50921 T66 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 50921 T78 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 50929 T74 oasc.ZkController.register We are http://127.0.0.1:48159/collection1/ and leader is http://127.0.0.1:48159/collection1/
   [junit4]   2> 50938 T74 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:48159
   [junit4]   2> 50939 T74 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 50939 T74 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 50939 T74 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 50942 T66 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 50944 T67 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48159",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:48159_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1"} current state version: 3
   [junit4]   2> 50945 T67 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48159",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:48159_",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 50946 T67 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 50949 T66 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 50949 T78 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 51059 T66 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 51059 T78 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 52547 T39 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-1-001/cores/collection1
   [junit4]   2> 52550 T39 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-1-001
   [junit4]   2> 52551 T39 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 52553 T39 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:24241
   [junit4]   2> 52554 T39 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/tempDir-001/jetty1, hostPort=24240, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-1-001/cores, hostContext=/, solrconfig=solrconfig.xml}
   [junit4]   2> 52554 T39 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 52555 T39 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-1-001/'
   [junit4]   2> 52603 T39 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-1-001/solr.xml
   [junit4]   2> 52665 T39 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-1-001/cores
   [junit4]   2> 52666 T39 oasc.CoreContainer.<init> New CoreContainer 1867869617
   [junit4]   2> 52666 T39 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-1-001/]
   [junit4]   2> 52666 T39 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-1-001/lib
   [junit4]   2> 52667 T39 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-1-001/lib).
   [junit4]   2> 52679 T39 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 52679 T39 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 52679 T39 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 52680 T39 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 52680 T39 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 52680 T39 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 52681 T39 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 52681 T39 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 52681 T39 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 52682 T39 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 52682 T39 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 52683 T39 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 52683 T39 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 52683 T39 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 52684 T39 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 52684 T39 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:56160/solr
   [junit4]   2> 52684 T39 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 52685 T39 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 52685 T39 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 52688 T93 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2c1c6bd9 name:ZooKeeperConnection Watcher:127.0.0.1:56160 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 52689 T39 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 52690 T39 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 52692 T39 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 52694 T96 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4f3830c9 name:ZooKeeperConnection Watcher:127.0.0.1:56160/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 52694 T39 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 52701 T39 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 53711 T39 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:24240_
   [junit4]   2> 53713 T39 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:24240_
   [junit4]   2> 53717 T39 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 53727 T39 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-1-001/cores
   [junit4]   1> CORE DESCRIPTOR: {transient=false, coreNodeName=, collection=collection1, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-1-001/cores/collection1/, schema=schema.xml, config=solrconfig.xml, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-1-001/cores/collection1, shard=, dataDir=data/}
   [junit4]   2> 53730 T39 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-1-001/cores/collection1/
   [junit4]   2> 53730 T39 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 53734 T97 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 53734 T97 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 53736 T66 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 53737 T97 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 53738 T97 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 53738 T67 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:24240",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:24240_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null} current state version: 5
   [junit4]   2> 53738 T97 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 53739 T67 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:24240",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:24240_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 53740 T67 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 53740 T67 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 53743 T67 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 53748 T97 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 53748 T97 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 53748 T97 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 53749 T97 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 53749 T97 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 53750 T97 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 53751 T97 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 53751 T97 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-1-001/cores/collection1/'
   [junit4]   2> 53766 T97 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 53769 T97 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 53784 T97 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 53805 T97 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 53852 T97 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 53854 T97 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 53887 T97 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 54309 T97 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 54326 T97 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 54328 T97 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 54356 T97 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 54361 T97 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 54366 T97 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 54368 T97 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 54368 T97 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 54369 T97 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 54370 T97 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 54370 T97 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 54371 T97 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 54371 T97 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 54371 T97 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 54372 T97 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@56f3b98c
   [junit4]   2> 54373 T97 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-1-001/cores/collection1/data
   [junit4]   2> 54373 T97 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 54373 T97 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 54374 T97 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 54374 T97 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=15, maxMergeAtOnceExplicit=18, maxMergedSegmentMB=71.2314453125, floorSegmentMB=1.5166015625, forceMergeDeletesPctAllowed=23.887340780558468, segmentsPerTier=50.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7575247963883267
   [junit4]   2> 54375 T97 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@1534fa6c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4924d354),segFN=segments_1,generation=1}
   [junit4]   2> 54375 T97 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 54386 T97 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 54386 T97 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 54387 T97 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 54387 T97 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 54387 T97 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 54388 T97 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 54388 T97 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 54389 T97 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 54389 T97 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 54407 T97 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 54411 T97 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 54414 T97 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 54416 T97 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 54445 T97 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/ping,/admin/plugins,/admin/luke,/admin/properties,/admin/threads,/update/json/docs,/schema,/admin/logging,/config,/admin/system,/admin/segments,/admin/file,/update/csv,/get,/update/json,/admin/mbeans,standard,/replication,/update
   [junit4]   2> 54447 T97 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 54448 T97 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 54448 T97 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 54450 T97 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 54450 T97 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 54452 T97 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=35, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=0.78515625, noCFSRatio=1.0]
   [junit4]   2> 54452 T97 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@1534fa6c lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4924d354),segFN=segments_1,generation=1}
   [junit4]   2> 54453 T97 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 54453 T97 oass.SolrIndexSearcher.<init> Opening Searcher@39b64680[collection1] main
   [junit4]   2> 54453 T97 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 54455 T97 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 54456 T97 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 54456 T97 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 54457 T97 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 54457 T97 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 54458 T97 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 54458 T97 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 54459 T97 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 54459 T97 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 54461 T98 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@39b64680[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 54462 T97 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 54463 T101 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:24240 collection:collection1 shard:shard2
   [junit4]   2> 54464 T39 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 54464 T39 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 54465 T101 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 54475 T101 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 54478 T66 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 54478 T101 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 54479 T101 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C10 name=collection1 org.apache.solr.core.SolrCore@10632e39 url=http://127.0.0.1:24240/collection1 node=127.0.0.1:24240_ C10_STATE=coll:collection1 core:collection1 props:{core=collection1, base_url=http://127.0.0.1:24240, node_name=127.0.0.1:24240_, state=down}
   [junit4]   2> 54479 T101 C10 P24240 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:24240/collection1/
   [junit4]   2> 54479 T101 C10 P24240 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 54480 T101 C10 P24240 oasc.SyncStrategy.syncToMe http://127.0.0.1:24240/collection1/ has no replicas
   [junit4]   2> 54480 T101 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:24240/collection1/ shard2
   [junit4]   2> 54480 T67 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 54480 T101 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 54483 T67 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 54485 T96 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 54487 T96 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 54493 T67 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:24240",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 54497 T67 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 54499 T96 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 54500 T96 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 54507 T66 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 54541 T101 oasc.ZkController.register We are http://127.0.0.1:24240/collection1/ and leader is http://127.0.0.1:24240/collection1/
   [junit4]   2> 54542 T101 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:24240
   [junit4]   2> 54542 T101 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 54543 T101 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 54543 T101 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 54546 T66 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 54548 T67 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:24240",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:24240_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard2"} current state version: 5
   [junit4]   2> 54549 T67 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:24240",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:24240_",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard2"}
   [junit4]   2> 54553 T67 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 3
   [junit4]   2> 54553 T101 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 54554 T96 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 54556 T96 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 55734 T39 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-2-001/cores/collection1
   [junit4]   2> 55737 T39 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-2-001
   [junit4]   2> 55738 T39 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 55741 T39 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:24249
   [junit4]   2> 55741 T39 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/tempDir-001/jetty2, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-2-001/cores, hostPort=24248, solrconfig=solrconfig.xml}
   [junit4]   2> 55742 T39 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@631d75b9
   [junit4]   2> 55742 T39 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-2-001/'
   [junit4]   2> 55778 T39 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-2-001/solr.xml
   [junit4]   2> 55823 T39 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-2-001/cores
   [junit4]   2> 55824 T39 oasc.CoreContainer.<init> New CoreContainer 1408203680
   [junit4]   2> 55824 T39 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-2-001/]
   [junit4]   2> 55825 T39 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-2-001/lib
   [junit4]   2> 55825 T39 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 18C2A3D48792D947-001/shard-2-001/lib).
   [junit4]   2> 55838 T39 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 55838 T39 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 55839 T39 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 55839 T39 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 55840 T39 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 55840 T39 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 55840 T39 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 55841 T39 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 55841 T39 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 55842 T39 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 55842 T39 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 55843 T39 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 55844 T39 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 55844 T39 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 55844 T39 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 55845 T39 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:56160/solr
   [junit4]   2> 55845 T39 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 55845 T39 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 55846 T39 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 55848 T114 oascc.ConnectionManager.process Watcher org

[...truncated too long message...]

  2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 2590428 T4963 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:64366/
   [junit4]   2> 2590428 T4963 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:64343/
   [junit4]   2> 2590429 T4963 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:64353/
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=18C2A3D48792D947 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=en_IE -Dtests.timezone=America/Noronha -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   65.8s J3 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:64353/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([18C2A3D48792D947:90969C0E296EB4BF]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:64353/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	... 54 more
   [junit4]   2> 2590443 T4963 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 18C2A3D48792D947-001
   [junit4]   2> 65801 T4962 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {multiDefault=PostingsFormat(name=LuceneVarGapDocFreqInterval), _version_=PostingsFormat(name=Memory doPackFST= true), range_facet_l=PostingsFormat(name=LuceneVarGapDocFreqInterval), intDefault=PostingsFormat(name=Memory doPackFST= true), a_t=PostingsFormat(name=LuceneVarGapDocFreqInterval), text=PostingsFormat(name=Memory doPackFST= false), timestamp=PostingsFormat(name=LuceneVarGapDocFreqInterval), id=PostingsFormat(name=Asserting)}, docValues:{timestamp=DocValuesFormat(name=Direct)}, sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=en_IE, timezone=America/Noronha
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=175405632,total=323485696
   [junit4]   2> NOTE: All tests run in this JVM: [TestQuerySenderListener, TestRangeQuery, TestPivotHelperCode, ExternalCollectionsTest, TermsComponentTest, DocValuesTest, TermVectorComponentDistributedTest, DistributedDebugComponentTest, SolrCoreTest, TestCodecSupport, TestSystemIdResolver, MultiTermTest, SuggesterWFSTTest, TestIntervalFaceting, TestCloudManagedSchema, OverseerRolesTest, TestArbitraryIndexDir, TestFiltering, TestValueSourceCache, AnalyticsMergeStrategyTest, CurrencyFieldXmlFileTest, DateMathParserTest, TestIndexSearcher, TestStressUserVersions, TestUtils, OutputWriterTest, TestWordDelimiterFilterFactory, TestSolrQueryParserDefaultOperatorResource, TestSchemaManager, TestGroupingSearch, TestHighlightDedupGrouping, RequestLoggingTest, DeleteShardTest, DistanceUnitsTest, FileUtilsTest, ConnectionManagerTest, DocValuesMultiTest, TestExpandComponent, TestSolrConfigHandlerConcurrent, AlternateDirectoryTest, TestManagedSchemaFieldTypeResource, StressHdfsTest, OutOfBoxZkACLAndCredentialsProvidersTest, UUIDUpdateProcessorFallbackTest, TestWriterPerf, DistributedFacetPivotLargeTest, TestCSVLoader, CSVRequestHandlerTest, FacetPivotSmallTest, TestDefaultSimilarityFactory, DistribDocExpirationUpdateProcessorTest, TestDefaultSearchFieldResource, TestCollapseQParserPlugin, UpdateRequestProcessorFactoryTest, OverriddenZkACLAndCredentialsProvidersTest, SolrXmlInZkTest, TestDynamicFieldResource, DefaultValueUpdateProcessorTest, BasicDistributedZk2Test, UnloadDistributedZkTest, SyncSliceTest, TestReplicationHandler, LeaderInitiatedRecoveryOnCommitTest, MultiThreadedOCPTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J3 in 66.33s, 1 test, 1 error <<< FAILURES!

[...truncated 298 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 474 suites, 1890 tests, 3 errors, 58 ignored (9 assumptions)

Total time: 157 minutes 13 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 826260355 bytes
Compression is 0.0%
Took 2 min 34 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 790 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/790/

4 tests failed.
REGRESSION:  org.apache.solr.search.mlt.SimpleMLTQParserTest.doTest

Error Message:
Exception during query

Stack Trace:
java.lang.RuntimeException: Exception during query
	at __randomizedtesting.SeedInfo.seed([F395E0474DFB7C69:54D158E320406FD0]:0)
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:794)
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:761)
	at org.apache.solr.search.mlt.SimpleMLTQParserTest.doTest(SimpleMLTQParserTest.java:71)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result/doc[2]/int[@name='id'][.='13']
	xml response was: <?xml version="1.0" encoding="UTF-8"?>
<response>
<lst name="responseHeader"><int name="status">0</int><int name="QTime">7</int></lst><result name="response" numFound="14" start="0"><doc><int name="id">17</int><arr name="range_facet_l"><long>17</long></arr><arr name="lowerfilt"><str>The quote red fox jumped moon over the lazy brown dogs moon. Of course moon. Foxes and moon come back to the foxes and moon</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped moon over the lazy brown dogs moon. Of course moon. Foxes and moon come back to the foxes and moon</str></arr><long name="_version_">1495996036946067456</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.88Z</date></doc><doc><int name="id">14</int><arr name="range_facet_l"><long>14</long></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996036937678848</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.873Z</date></doc><doc><int name="id">13</int><arr name="range_facet_l"><long>13</long></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996036933484544</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.869Z</date></doc><doc><int name="id">18</int><arr name="range_facet_l"><long>18</long></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996036991156224</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.924Z</date></doc><doc><int name="id">20</int><arr name="range_facet_l"><long>20</long></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996036998496256</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.93Z</date></doc><doc><int name="id">22</int><arr name="range_facet_l"><long>22</long></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996037021564928</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.952Z</date></doc><doc><int name="id">23</int><arr name="range_facet_l"><long>23</long></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996037025759232</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.957Z</date></doc><doc><int name="id">9</int><arr name="range_facet_l"><long>9</long></arr><arr name="lowerfilt"><str>The quick red fox jumped over the lazy big and large brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quick red fox jumped over the lazy big and large brown dogs.</str></arr><long name="_version_">1495996036927193088</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.863Z</date></doc><doc><int name="id">7</int><arr name="range_facet_l"><long>7</long></arr><arr name="lowerfilt"><str>mclaren moon or the moon and moon moon shine and the moon but moon was good foxes too</str></arr><arr name="lowerfilt1and2"><str>mclaren moon or the moon and moon moon shine and the moon but moon was good foxes too</str></arr><long name="_version_">1495996036920901632</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.857Z</date></doc><doc><int name="id">15</int><arr name="range_facet_l"><long>15</long></arr><arr name="lowerfilt"><str>The fat red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The fat red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996036940824576</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.876Z</date></doc></result>
</response>

	request was:q=%7B%21mlt+qf%3Dlowerfilt%7D17
	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:787)
	... 41 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:57648/wymn/ta/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57648/wymn/ta/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([F395E0474DFB7C69:7BC1DF9DE3071191]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([F395E0474DFB7C69:7BC1DF9DE3071191]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([F395E0474DFB7C69:7BC1DF9DE3071191]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 10279 lines...]
   [junit4] Suite: org.apache.solr.search.mlt.SimpleMLTQParserTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.search.mlt.SimpleMLTQParserTest F395E0474DFB7C69-001/init-core-data-001
   [junit4]   2> 1032391 T34266 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 1032392 T34266 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> 1032393 T34266 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 1032393 T34266 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 1032394 T34266 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 1032394 T34266 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 1032483 T34266 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1032518 T34266 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 1032567 T34266 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1032568 T34266 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 1032576 T34266 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 1032856 T34266 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1032868 T34266 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1032870 T34266 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1032877 T34266 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1032880 T34266 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1032883 T34266 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1032884 T34266 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1032884 T34266 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1032885 T34266 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1032885 T34266 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1032885 T34266 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1032886 T34266 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 1032886 T34266 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr
   [junit4]   2> 1032886 T34266 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/'
   [junit4]   2> 1032897 T34266 oasc.CoreContainer.<init> New CoreContainer 470313703
   [junit4]   2> 1032898 T34266 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/]
   [junit4]   2> 1032898 T34266 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/lib
   [junit4]   2> 1032898 T34266 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/lib).
   [junit4]   2> 1032907 T34266 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 600000
   [junit4]   2> 1032907 T34266 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 1032907 T34266 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 60000
   [junit4]   2> 1032907 T34266 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1032907 T34266 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1032908 T34266 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1032908 T34266 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1032908 T34266 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1032908 T34266 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1032908 T34266 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1032909 T34266 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1032909 T34266 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=30000&connTimeout=30000&retry=true
   [junit4]   2> 1032909 T34266 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1032909 T34266 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1032910 T34266 oasc.CoreContainer.load Node Name: testNode
   [junit4]   1> CORE DESCRIPTOR: {instanceDir=collection1, shard=shard1, collection=collection1, transient=false, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/, schema=schema.xml, name=collection1, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.search.mlt.SimpleMLTQParserTest F395E0474DFB7C69-001/init-core-data-001, loadOnStartup=true, config=solrconfig.xml}
   [junit4]   2> 1032912 T34267 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 1032913 T34267 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 1032914 T34267 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 1032914 T34267 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 1032954 T34267 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1032984 T34267 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 1033054 T34267 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1033055 T34267 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 1033065 T34267 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1033447 T34267 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1033458 T34267 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1033461 T34267 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1033470 T34267 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1033474 T34267 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1033478 T34267 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1033478 T34267 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1033478 T34267 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1033479 T34267 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1033479 T34267 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1033479 T34267 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1033480 T34267 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 1033480 T34267 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1033480 T34267 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.search.mlt.SimpleMLTQParserTest F395E0474DFB7C69-001/init-core-data-001/
   [junit4]   2> 1033480 T34267 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@2a1aa6a
   [junit4]   2> 1033481 T34267 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.search.mlt.SimpleMLTQParserTest F395E0474DFB7C69-001/init-core-data-001
   [junit4]   2> 1033482 T34267 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.search.mlt.SimpleMLTQParserTest F395E0474DFB7C69-001/init-core-data-001/index/
   [junit4]   2> 1033482 T34267 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.search.mlt.SimpleMLTQParserTest F395E0474DFB7C69-001/init-core-data-001/index' doesn't exist. Creating new index...
   [junit4]   2> 1033482 T34267 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.search.mlt.SimpleMLTQParserTest F395E0474DFB7C69-001/init-core-data-001/index
   [junit4]   2> 1033482 T34267 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=15, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4906423311474368]
   [junit4]   2> 1033483 T34267 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@46e91c55 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@724334d1),segFN=segments_1,generation=1}
   [junit4]   2> 1033484 T34267 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1033490 T34267 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1033490 T34267 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1033490 T34267 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 1033490 T34267 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 1033491 T34267 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1033491 T34267 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1033491 T34267 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 1033492 T34267 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 1033492 T34267 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1033492 T34267 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1033493 T34267 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1033493 T34267 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1033519 T34267 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1033522 T34267 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1033524 T34267 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1033526 T34267 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1033532 T34267 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 1033535 T34267 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/properties,mltrh,spellCheckCompRH,/admin/system,/debug/dump,/schema,standard,/update/csv,/admin/file,/update,dismax,/admin/plugins,spellCheckWithWordbreak_Direct,/mlt,lazy,spellCheckCompRH_Direct,spellCheckCompRH1,/admin/mbeans,/admin/segments,mock,/terms,/admin/logging,/replication,/config,/admin/ping,/admin/luke,/update/json/docs,tvrh,/admin/threads,defaults,/search-facet-def,/get,spellCheckWithWordbreak,/search-facet-invariants,/update/json
   [junit4]   2> 1033537 T34267 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 1033538 T34267 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1033540 T34267 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1033540 T34267 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1033540 T34267 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1033541 T34267 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1033541 T34267 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@5bdfeb97
   [junit4]   2> 1033542 T34267 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@46e91c55 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@724334d1),segFN=segments_1,generation=1}
   [junit4]   2> 1033542 T34267 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1033542 T34267 oass.SolrIndexSearcher.<init> Opening Searcher@3ae24e5[collection1] main
   [junit4]   2> 1033543 T34267 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 1033543 T34267 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 1033543 T34267 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1033544 T34267 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 1033564 T34267 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 1033607 T34267 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1033607 T34268 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 1033608 T34268 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 1033608 T34268 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 1033608 T34268 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 1033609 T34267 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1033609 T34268 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 1033610 T34268 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 1033611 T34266 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 1033611 T34268 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 1033612 T34268 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 1033613 T34268 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 1033615 T34266 oas.SolrTestCaseJ4.setUp ###Starting doTest
   [junit4]   2> 1033615 T34268 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3ae24e5[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> ASYNC  NEW_CORE C907 name=collection1 org.apache.solr.core.SolrCore@4839082c
   [junit4]   2> 1033619 T34266 C907 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[1 (1495996036902027264)]} 0 3
   [junit4]   2> 1033623 T34266 C907 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[2 (1495996036908318720)]} 0 1
   [junit4]   2> 1033624 T34266 C907 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[3 (1495996036910415872)]} 0 0
   [junit4]   2> 1033629 T34266 C907 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[4 (1495996036912513024)]} 0 4
   [junit4]   2> 1033631 T34266 C907 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[5 (1495996036916707328)]} 0 1
   [junit4]   2> 1033633 T34266 C907 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[6 (1495996036918804480)]} 0 1
   [junit4]   2> 1033635 T34266 C907 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[7 (1495996036920901632)]} 0 1
   [junit4]   2> 1033639 T34266 C907 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[8 (1495996036922998784)]} 0 3
   [junit4]   2> 1033642 T34266 C907 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[9 (1495996036927193088)]} 0 2
   [junit4]   2> 1033643 T34266 C907 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[10 (1495996036930338816)]} 0 0
   [junit4]   2> 1033645 T34266 C907 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[12 (1495996036931387392)]} 0 1
   [junit4]   2> 1033649 T34266 C907 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[13 (1495996036933484544)]} 0 3
   [junit4]   2> 1033652 T34266 C907 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[14 (1495996036937678848)]} 0 2
   [junit4]   2> 1033654 T34266 C907 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[15 (1495996036940824576)]} 0 1
   [junit4]   2> 1033655 T34266 C907 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[16 (1495996036942921728)]} 0 0
   [junit4]   2> 1033699 T34266 C907 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[17 (1495996036946067456)]} 0 42
   [junit4]   2> 1033703 T34266 C907 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[18 (1495996036991156224)]} 0 2
   [junit4]   2> 1033706 T34266 C907 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[19 (1495996036995350528)]} 0 1
   [junit4]   2> 1033708 T34266 C907 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[20 (1495996036998496256)]} 0 1
   [junit4]   2> 1033727 T34266 C907 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[21 (1495996037000593408)]} 0 17
   [junit4]   2> 1033732 T34266 C907 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[22 (1495996037021564928)]} 0 3
   [junit4]   2> 1033735 T34266 C907 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[23 (1495996037025759232)]} 0 1
   [junit4]   2> 1033738 T34266 C907 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[24 (1495996037028904960)]} 0 1
   [junit4]   2> 1033750 T34266 C907 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[25 (1495996037031002112)]} 0 11
   [junit4]   2> 1033752 T34266 C907 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 1033756 T34266 C907 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@46e91c55 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@724334d1),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@46e91c55 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@724334d1),segFN=segments_2,generation=2}
   [junit4]   2> 1033757 T34266 C907 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 1033758 T34266 C907 oass.SolrIndexSearcher.<init> Opening Searcher@3e53d368[collection1] main
   [junit4]   2> 1033759 T34266 C907 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 1033760 T34268 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3e53d368[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_6(5.1.0):C20) Uninverting(_7(5.1.0):c4)))}
   [junit4]   2> 1033761 T34266 C907 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {commit=} 0 9
   [junit4]   2> 1033770 T34266 C907 oasc.SolrCore.execute [collection1] webapp=null path=null params={q=%7B%21mlt+qf%3Dlowerfilt%7D17} hits=14 status=0 QTime=7 
   [junit4]   2> 1033774 T34266 oas.SolrTestCaseJ4.assertQ ERROR REQUEST FAILED: xpath=//result/doc[2]/int[@name='id'][.='13']
   [junit4]   2> 		xml response was: <?xml version="1.0" encoding="UTF-8"?>
   [junit4]   2> 	<response>
   [junit4]   2> 	<lst name="responseHeader"><int name="status">0</int><int name="QTime">7</int></lst><result name="response" numFound="14" start="0"><doc><int name="id">17</int><arr name="range_facet_l"><long>17</long></arr><arr name="lowerfilt"><str>The quote red fox jumped moon over the lazy brown dogs moon. Of course moon. Foxes and moon come back to the foxes and moon</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped moon over the lazy brown dogs moon. Of course moon. Foxes and moon come back to the foxes and moon</str></arr><long name="_version_">1495996036946067456</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.88Z</date></doc><doc><int name="id">14</int><arr name="range_facet_l"><long>14</long></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996036937678848</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.873Z</date></doc><doc><int name="id">13</int><arr name="range_facet_l"><long>13</long></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996036933484544</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.869Z</date></doc><doc><int name="id">18</int><arr name="range_facet_l"><long>18</long></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996036991156224</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.924Z</date></doc><doc><int name="id">20</int><arr name="range_facet_l"><long>20</long></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996036998496256</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.93Z</date></doc><doc><int name="id">22</int><arr name="range_facet_l"><long>22</long></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996037021564928</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.952Z</date></doc><doc><int name="id">23</int><arr name="range_facet_l"><long>23</long></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996037025759232</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.957Z</date></doc><doc><int name="id">9</int><arr name="range_facet_l"><long>9</long></arr><arr name="lowerfilt"><str>The quick red fox jumped over the lazy big and large brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quick red fox jumped over the lazy big and large brown dogs.</str></arr><long name="_version_">1495996036927193088</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.863Z</date></doc><doc><int name="id">7</int><arr name="range_facet_l"><long>7</long></arr><arr name="lowerfilt"><str>mclaren moon or the moon and moon moon shine and the moon but moon was good foxes too</str></arr><arr name="lowerfilt1and2"><str>mclaren moon or the moon and moon moon shine and the moon but moon was good foxes too</str></arr><long name="_version_">1495996036920901632</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.857Z</date></doc><doc><int name="id">15</int><arr name="range_facet_l"><long>15</long></arr><arr name="lowerfilt"><str>The fat red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The fat red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996036940824576</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.876Z</date></doc></result>
   [junit4]   2> 	</response>
   [junit4]   2> 	
   [junit4]   2> 		request was:q=%7B%21mlt+qf%3Dlowerfilt%7D17
   [junit4]   2> 1033777 T34266 oasc.SolrException.log ERROR REQUEST FAILED: q=%7B%21mlt+qf%3Dlowerfilt%7D17:java.lang.RuntimeException: REQUEST FAILED: xpath=//result/doc[2]/int[@name='id'][.='13']
   [junit4]   2> 		xml response was: <?xml version="1.0" encoding="UTF-8"?>
   [junit4]   2> 	<response>
   [junit4]   2> 	<lst name="responseHeader"><int name="status">0</int><int name="QTime">7</int></lst><result name="response" numFound="14" start="0"><doc><int name="id">17</int><arr name="range_facet_l"><long>17</long></arr><arr name="lowerfilt"><str>The quote red fox jumped moon over the lazy brown dogs moon. Of course moon. Foxes and moon come back to the foxes and moon</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped moon over the lazy brown dogs moon. Of course moon. Foxes and moon come back to the foxes and moon</str></arr><long name="_version_">1495996036946067456</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.88Z</date></doc><doc><int name="id">14</int><arr name="range_facet_l"><long>14</long></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996036937678848</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.873Z</date></doc><doc><int name="id">13</int><arr name="range_facet_l"><long>13</long></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996036933484544</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.869Z</date></doc><doc><int name="id">18</int><arr name="range_facet_l"><long>18</long></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996036991156224</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.924Z</date></doc><doc><int name="id">20</int><arr name="range_facet_l"><long>20</long></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996036998496256</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.93Z</date></doc><doc><int name="id">22</int><arr name="range_facet_l"><long>22</long></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996037021564928</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.952Z</date></doc><doc><int name="id">23</int><arr name="range_facet_l"><long>23</long></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996037025759232</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.957Z</date></doc><doc><int name="id">9</int><arr name="range_facet_l"><long>9</long></arr><arr name="lowerfilt"><str>The quick red fox jumped over the lazy big and large brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quick red fox jumped over the lazy big and large brown dogs.</str></arr><long name="_version_">1495996036927193088</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.863Z</date></doc><doc><int name="id">7</int><arr name="range_facet_l"><long>7</long></arr><arr name="lowerfilt"><str>mclaren moon or the moon and moon moon shine and the moon but moon was good foxes too</str></arr><arr name="lowerfilt1and2"><str>mclaren moon or the moon and moon moon shine and the moon but moon was good foxes too</str></arr><long name="_version_">1495996036920901632</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.857Z</date></doc><doc><int name="id">15</int><arr name="range_facet_l"><long>15</long></arr><arr name="lowerfilt"><str>The fat red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The fat red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996036940824576</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.876Z</date></doc></result>
   [junit4]   2> 	</response>
   [junit4]   2> 	
   [junit4]   2> 		request was:q=%7B%21mlt+qf%3Dlowerfilt%7D17
   [junit4]   2> 		at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:787)
   [junit4]   2> 		at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:761)
   [junit4]   2> 		at org.apache.solr.search.mlt.SimpleMLTQParserTest.doTest(SimpleMLTQParserTest.java:71)
   [junit4]   2> 		at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
   [junit4]   2> 		at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
   [junit4]   2> 		at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
   [junit4]   2> 		at java.lang.reflect.Method.invoke(Method.java:606)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
   [junit4]   2> 		at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
   [junit4]   2> 		at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
   [junit4]   2> 		at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
   [junit4]   2> 		at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
   [junit4]   2> 		at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
   [junit4]   2> 		at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 		at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 		at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
   [junit4]   2> 		at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
   [junit4]   2> 		at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
   [junit4]   2> 		at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
   [junit4]   2> 		at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
   [junit4]   2> 		at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 	
   [junit4]   2> 1033779 T34266 oas.SolrTestCaseJ4.tearDown ###Ending doTest
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SimpleMLTQParserTest -Dtests.method=doTest -Dtests.seed=F395E0474DFB7C69 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=bg_BG -Dtests.timezone=Antarctica/Palmer -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.18s J2 | SimpleMLTQParserTest.doTest <<<
   [junit4]    > Throwable #1: java.lang.RuntimeException: Exception during query
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F395E0474DFB7C69:54D158E320406FD0]:0)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:794)
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:761)
   [junit4]    > 	at org.apache.solr.search.mlt.SimpleMLTQParserTest.doTest(SimpleMLTQParserTest.java:71)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: java.lang.RuntimeException: REQUEST FAILED: xpath=//result/doc[2]/int[@name='id'][.='13']
   [junit4]    > 	xml response was: <?xml version="1.0" encoding="UTF-8"?>
   [junit4]    > <response>
   [junit4]    > <lst name="responseHeader"><int name="status">0</int><int name="QTime">7</int></lst><result name="response" numFound="14" start="0"><doc><int name="id">17</int><arr name="range_facet_l"><long>17</long></arr><arr name="lowerfilt"><str>The quote red fox jumped moon over the lazy brown dogs moon. Of course moon. Foxes and moon come back to the foxes and moon</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped moon over the lazy brown dogs moon. Of course moon. Foxes and moon come back to the foxes and moon</str></arr><long name="_version_">1495996036946067456</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.88Z</date></doc><doc><int name="id">14</int><arr name="range_facet_l"><long>14</long></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996036937678848</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.873Z</date></doc><doc><int name="id">13</int><arr name="range_facet_l"><long>13</long></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996036933484544</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.869Z</date></doc><doc><int name="id">18</int><arr name="range_facet_l"><long>18</long></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996036991156224</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.924Z</date></doc><doc><int name="id">20</int><arr name="range_facet_l"><long>20</long></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996036998496256</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.93Z</date></doc><doc><int name="id">22</int><arr name="range_facet_l"><long>22</long></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996037021564928</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.952Z</date></doc><doc><int name="id">23</int><arr name="range_facet_l"><long>23</long></arr><arr name="lowerfilt"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quote red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996037025759232</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.957Z</date></doc><doc><int name="id">9</int><arr name="range_facet_l"><long>9</long></arr><arr name="lowerfilt"><str>The quick red fox jumped over the lazy big and large brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The quick red fox jumped over the lazy big and large brown dogs.</str></arr><long name="_version_">1495996036927193088</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.863Z</date></doc><doc><int name="id">7</int><arr name="range_facet_l"><long>7</long></arr><arr name="lowerfilt"><str>mclaren moon or the moon and moon moon shine and the moon but moon was good foxes too</str></arr><arr name="lowerfilt1and2"><str>mclaren moon or the moon and moon moon shine and the moon but moon was good foxes too</str></arr><long name="_version_">1495996036920901632</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.857Z</date></doc><doc><int name="id">15</int><arr name="range_facet_l"><long>15</long></arr><arr name="lowerfilt"><str>The fat red fox jumped over the lazy brown dogs.</str></arr><arr name="lowerfilt1and2"><str>The fat red fox jumped over the lazy brown dogs.</str></arr><long name="_version_">1495996036940824576</long><arr name="multiDefault"><str>muLti-Default</str></arr><int name="intDefault">42</int><date name="timestamp">2015-03-18T15:36:37.876Z</date></doc></result>
   [junit4]    > </response>
   [junit4]    > 	request was:q=%7B%21mlt+qf%3Dlowerfilt%7D17
   [junit4]    > 	at org.apache.solr.SolrTestCaseJ4.assertQ(SolrTestCaseJ4.java:787)
   [junit4]    > 	... 41 more
   [junit4]   2> 1033792 T34266 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> 1033792 T34266 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=470313703
   [junit4]   2> 1033792 T34266 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@4839082c
   [junit4]   2> 1033793 T34266 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=1,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=24,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=1864,transaction_logs_total_number=1}
   [junit4]   2> 1033793 T34266 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 1033793 T34266 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 1033794 T34266 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 1033795 T34266 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 1033819 T34266 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 1033820 T34266 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.search.mlt.SimpleMLTQParserTest F395E0474DFB7C69-001/init-core-data-001/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.search.mlt.SimpleMLTQParserTest F395E0474DFB7C69-001/init-core-data-001/index;done=false>>]
   [junit4]   2> 1033820 T34266 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.search.mlt.SimpleMLTQParserTest F395E0474DFB7C69-001/init-core-data-001/index
   [junit4]   2> 1033821 T34266 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.search.mlt.SimpleMLTQParserTest F395E0474DFB7C69-001/init-core-data-001 [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.search.mlt.SimpleMLTQParserTest F395E0474DFB7C69-001/init-core-data-001;done=false>>]
   [junit4]   2> 1033821 T34266 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.search.mlt.SimpleMLTQParserTest F395E0474DFB7C69-001/init-core-data-001
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.search.mlt.SimpleMLTQParserTest F395E0474DFB7C69-001
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=DefaultSimilarity, locale=bg_BG, timezone=Antarctica/Palmer
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=138850704,total=346030080
   [junit4]   2> NOTE: All tests run in this JVM: [DocumentBuilderTest, TestManagedStopFilterFactory, LukeRequestHandlerTest, FileBasedSpellCheckerTest, HdfsChaosMonkeySafeLeaderTest, CurrencyFieldOpenExchangeTest, SolrCoreCheckLockOnStartupTest, DirectUpdateHandlerTest, CoreMergeIndexesAdminHandlerTest, TestReRankQParserPlugin, TestFieldSortValues, SolrPluginUtilsTest, SaslZkACLProviderTest, TestExceedMaxTermLength, SpatialFilterTest, TestQuerySenderNoQuery, TestReplicationHandlerBackup, TestMergePolicyConfig, StandardRequestHandlerTest, DirectUpdateHandlerOptimizeTest, TestExtendedDismaxParser, SliceStateTest, TestBulkSchemaConcurrent, TestDynamicLoading, TestManagedResource, TestPerFieldSimilarity, SOLR749Test, TestRemoteStreaming, MoreLikeThisHandlerTest, InfoHandlerTest, DeleteLastCustomShardedReplicaTest, TestRebalanceLeaders, TestRestManager, TestObjectReleaseTracker, URLClassifyProcessorTest, TestComplexPhraseQParserPlugin, SolrRequestParserTest, CacheHeaderTest, TestReload, TestDistributedSearch, TestHdfsUpdateLog, TestPHPSerializedResponseWriter, TestRawResponseWriter, TestBM25SimilarityFactory, SimpleMLTQParserTest]
   [junit4] Completed on J2 in 1.46s, 1 test, 1 error <<< FAILURES!

[...truncated 246 lines...]
   [junit4] Suite: org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F395E0474DFB7C69-001/init-core-data-001
   [junit4]   2> 1257899 T2917 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /wymn/ta
   [junit4]   2> 1257906 T2917 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1257906 T2918 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1258006 T2917 oasc.ZkTestServer.run start zk server on port:57501
   [junit4]   2> 1258007 T2917 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1258008 T2917 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1258013 T2925 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@386e5a6c name:ZooKeeperConnection Watcher:127.0.0.1:57501 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1258014 T2917 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1258014 T2917 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1258014 T2917 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1258017 T2917 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1258018 T2917 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1258019 T2928 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@26688464 name:ZooKeeperConnection Watcher:127.0.0.1:57501/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1258020 T2917 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1258020 T2917 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1258020 T2917 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1258022 T2917 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1258024 T2917 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1258025 T2917 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1258027 T2917 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1258028 T2917 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1258031 T2917 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1258031 T2917 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1258134 T2917 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1258135 T2917 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1258137 T2917 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1258138 T2917 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1258140 T2917 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1258140 T2917 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1258142 T2917 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1258143 T2917 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1258145 T2917 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1258146 T2917 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1258147 T2917 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1258148 T2917 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1258150 T2917 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1258151 T2917 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1258153 T2917 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1258153 T2917 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1258155 T2917 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1258156 T2917 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1260082 T2917 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F395E0474DFB7C69-001/control-001/cores/collection1
   [junit4]   2> 1260086 T2917 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1260092 T2917 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:57508
   [junit4]   2> 1260092 T2917 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F395E0474DFB7C69-001/control-001/cores, hostPort=57507, hostContext=/wymn/ta, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F395E0474DFB7C69-001/tempDir-001/control/data}
   [junit4]   2> 1260093 T2917 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 1260093 T2917 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F395E0474DFB7C69-001/control-001/'
   [junit4]   2> 1260144 T2917 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F395E0474DFB7C69-001/control-001/solr.xml
   [junit4]   2> 1260207 T2917 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.LeaderInitiatedRecoveryOnCommitTest F395E0474DFB7C69-001/control-001/c

[...truncated too long message...]

rstate.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 		2	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 2220010 T10235 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:35542/_tw/t
   [junit4]   2> 2220010 T10235 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:35559/_tw/t
   [junit4]   2> 2220010 T10235 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:35548/_tw/t
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=F395E0474DFB7C69 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=it -Dtests.timezone=Europe/Gibraltar -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   61.5s J3 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([F395E0474DFB7C69:7BC1DF9DE3071191]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 2220028 T10235 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest F395E0474DFB7C69-001
   [junit4]   2> 61546 T10234 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Lucene50, sim=DefaultSimilarity, locale=it, timezone=Europe/Gibraltar
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=200360448,total=496500736
   [junit4]   2> NOTE: All tests run in this JVM: [TermsComponentTest, TestCollapseQParserPlugin, TestQueryTypes, IgnoreCommitOptimizeUpdateProcessorFactoryTest, TestSolrQueryParserResource, SharedFSAutoReplicaFailoverTest, OverseerRolesTest, TestFiltering, TestIndexingPerformance, ResponseLogComponentTest, TestCSVLoader, TermVectorComponentDistributedTest, SolrXmlInZkTest, ClusterStateUpdateTest, UnloadDistributedZkTest, TestDistributedGrouping, DefaultValueUpdateProcessorTest, TestWordDelimiterFilterFactory, DistributedMLTComponentTest, BlockCacheTest, DistanceUnitsTest, TestSystemIdResolver, SampleTest, TestCodecSupport, TestValueSourceCache, TestClusterStateMutator, TestManagedSchemaFieldTypeResource, AnalysisAfterCoreReloadTest, TestJoin, AlternateDirectoryTest, AnalyticsMergeStrategyTest, SoftAutoCommitTest, CollectionsAPIDistributedZkTest, BJQParserTest, CSVRequestHandlerTest, DocValuesMultiTest, DistribDocExpirationUpdateProcessorTest, TestShardHandlerFactory, TestIntervalFaceting, OverriddenZkACLAndCredentialsProvidersTest, BinaryUpdateRequestHandlerTest, ScriptEngineTest, TestIndexSearcher, OutOfBoxZkACLAndCredentialsProvidersTest, FacetPivotSmallTest, ExternalCollectionsTest, DistributedSuggestComponentTest, TestStressUserVersions, TestManagedSchema, SpellPossibilityIteratorTest, TestOrdValues, TestSolrConfigHandlerConcurrent, ZkCLITest, RequestLoggingTest, TestDynamicFieldResource, TestLMDirichletSimilarityFactory, TestPseudoReturnFields, SolrCoreTest, ChaosMonkeySafeLeaderTest, TestStressLucene, SuggesterWFSTTest, QueryEqualityTest, XsltUpdateRequestHandlerTest, TestQueryUtils, TestOmitPositions, TermVectorComponentTest, LoggingHandlerTest, TestJmxIntegration, TestCSVResponseWriter, TestAnalyzedSuggestions, SolrIndexConfigTest, TestDocSet, NumericFieldsTest, TestElisionMultitermQuery, ExternalFileFieldSortTest, TestPostingsSolrHighlighter, TestLuceneMatchVersion, TestDFRSimilarityFactory, TestDefaultSimilarityFactory, TestJmxMonitoredMap, TestFastOutputStream, TestFastLRUCache, ChaosMonkeyNothingIsSafeTest, CollectionsAPIAsyncDistributedZkTest, ConnectionManagerTest, DeleteInactiveReplicaTest, DistribJoinFromCollectionTest, MultiThreadedOCPTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J3 in 61.84s, 1 test, 1 error <<< FAILURES!

[...truncated 336 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 474 suites, 1890 tests, 4 errors, 58 ignored (9 assumptions)

Total time: 154 minutes 52 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 801980473 bytes
Compression is 0.0%
Took 2 min 29 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 789 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/789/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:17957/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:17957/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([8381C26FCFFDE0A1:BD5FDB561018D59]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([8381C26FCFFDE0A1:BD5FDB561018D59]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([8381C26FCFFDE0A1:BD5FDB561018D59]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 10425 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/init-core-data-001
   [junit4]   2> 1298782 T3630 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /c_/n
   [junit4]   2> 1298789 T3630 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 1298790 T3631 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 1298890 T3630 oasc.ZkTestServer.run start zk server on port:18957
   [junit4]   2> 1298891 T3630 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1298892 T3630 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1298897 T3638 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3a8e8916 name:ZooKeeperConnection Watcher:127.0.0.1:18957 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1298897 T3630 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1298897 T3630 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1298898 T3630 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 1298900 T3630 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1298901 T3630 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1298903 T3641 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@49e3c6d name:ZooKeeperConnection Watcher:127.0.0.1:18957/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1298903 T3630 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1298903 T3630 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1298904 T3630 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 1298906 T3630 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 1298907 T3630 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 1298909 T3630 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 1298910 T3630 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 1298911 T3630 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 1298914 T3630 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 1298914 T3630 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 1299260 T3630 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1299260 T3630 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 1299262 T3630 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 1299263 T3630 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 1299264 T3630 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 1299265 T3630 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 1299266 T3630 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 1299266 T3630 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 1299268 T3630 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 1299268 T3630 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 1299270 T3630 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1299270 T3630 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 1299271 T3630 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1299272 T3630 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 1299273 T3630 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 1299273 T3630 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 1299275 T3630 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 1299275 T3630 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 1300304 T3630 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/control-001/cores/collection1
   [junit4]   2> 1300308 T3630 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1300313 T3630 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:18963
   [junit4]   2> 1300313 T3630 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=18962, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/control-001/cores, hostContext=/c_/n}
   [junit4]   2> 1300314 T3630 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 1300314 T3630 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/control-001/'
   [junit4]   2> 1300360 T3630 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/control-001/solr.xml
   [junit4]   2> 1300416 T3630 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/control-001/cores
   [junit4]   2> 1300416 T3630 oasc.CoreContainer.<init> New CoreContainer 625037247
   [junit4]   2> 1300417 T3630 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/control-001/]
   [junit4]   2> 1300417 T3630 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/control-001/lib
   [junit4]   2> 1300418 T3630 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/control-001/lib).
   [junit4]   2> 1300431 T3630 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1300432 T3630 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1300432 T3630 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1300432 T3630 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1300433 T3630 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1300433 T3630 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1300434 T3630 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1300434 T3630 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1300434 T3630 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1300434 T3630 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1300435 T3630 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1300435 T3630 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1300436 T3630 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1300436 T3630 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1300436 T3630 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1300437 T3630 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:18957/solr
   [junit4]   2> 1300437 T3630 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1300437 T3630 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1300438 T3630 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1300440 T3653 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@16456cb7 name:ZooKeeperConnection Watcher:127.0.0.1:18957 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1300441 T3630 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1300441 T3630 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1300443 T3630 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1300444 T3656 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4743a2c9 name:ZooKeeperConnection Watcher:127.0.0.1:18957/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1300445 T3630 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1300446 T3630 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 1300448 T3630 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 1300450 T3630 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 1300452 T3630 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 1300454 T3630 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 1300456 T3630 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 1300457 T3630 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 1300459 T3630 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 1300460 T3630 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:18962_c_%2Fn
   [junit4]   2> 1300460 T3630 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:18962_c_%2Fn
   [junit4]   2> 1300462 T3630 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 1300463 T3630 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 1300464 T3630 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1300466 T3630 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:18962_c_%2Fn
   [junit4]   2> 1300466 T3630 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 1300467 T3630 oasc.Overseer.start Overseer (id=93494158004649987-127.0.0.1:18962_c_%2Fn-n_0000000000) starting
   [junit4]   2> 1300469 T3630 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 1300474 T3630 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 1300475 T3658 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 1300475 T3657 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 1300475 T3630 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1300478 T3630 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/control-001/cores
   [junit4]   1> CORE DESCRIPTOR: {instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/control-001/cores/collection1, dataDir=data/, collection=control_collection, shard=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/control-001/cores/collection1/, schema=schema.xml, coreNodeName=, name=collection1, transient=false, config=solrconfig.xml, loadOnStartup=true}
   [junit4]   2> 1300479 T3630 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/control-001/cores/collection1/
   [junit4]   2> 1300480 T3630 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1300480 T3660 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 1300480 T3660 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1300481 T3660 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1300481 T3656 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1300482 T3657 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:18962_c_%2Fn",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:18962/c_/n",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down"} current state version: 0
   [junit4]   2> 1300483 T3657 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:18962_c_%2Fn",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:18962/c_/n",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 1300483 T3657 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 1300483 T3657 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 1300484 T3656 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1301481 T3660 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1301482 T3660 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 1301483 T3660 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1301483 T3660 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1301484 T3660 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1301484 T3660 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/control-001/cores/collection1/'
   [junit4]   2> 1301502 T3660 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1301505 T3660 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1301516 T3660 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1301543 T3660 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 1301608 T3660 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1301709 T3660 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1301721 T3660 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1302143 T3660 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1302162 T3660 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1302165 T3660 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1302180 T3660 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1302186 T3660 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1302193 T3660 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1302194 T3660 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1302194 T3660 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1302195 T3660 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1302196 T3660 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1302196 T3660 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1302196 T3660 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 1302197 T3660 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1302197 T3660 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/control-001/cores/collection1/data/
   [junit4]   2> 1302197 T3660 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7280b7aa
   [junit4]   2> 1302198 T3660 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/control-001/cores/collection1/data
   [junit4]   2> 1302198 T3660 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/control-001/cores/collection1/data/index/
   [junit4]   2> 1302199 T3660 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1302199 T3660 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/control-001/cores/collection1/data/index
   [junit4]   2> 1302200 T3660 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=12, maxMergedSegmentMB=15.3505859375, floorSegmentMB=0.5556640625, forceMergeDeletesPctAllowed=27.798622184995335, segmentsPerTier=41.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3455216985199636
   [junit4]   2> 1302200 T3660 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@6cb48dfa lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@41d54fd7; maxCacheMB=0.115234375 maxMergeSizeMB=0.8251953125)),segFN=segments_1,generation=1}
   [junit4]   2> 1302201 T3660 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1302206 T3660 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1302206 T3660 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1302206 T3660 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1302207 T3660 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1302207 T3660 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1302207 T3660 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1302208 T3660 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1302208 T3660 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1302208 T3660 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1302237 T3660 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1302240 T3660 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1302242 T3660 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1302245 T3660 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1302249 T3660 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /replication,/admin/plugins,/admin/ping,/update/json/docs,/admin/mbeans,/schema,/update/json,/admin/system,/admin/logging,/update,/admin/threads,/admin/segments,standard,/admin/properties,/admin/luke,/get,/admin/file,/update/csv,/config
   [junit4]   2> 1302252 T3660 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1302254 T3660 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1302254 T3660 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1302256 T3660 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1302256 T3660 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1302257 T3660 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1205303418, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1302258 T3660 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@6cb48dfa lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@41d54fd7; maxCacheMB=0.115234375 maxMergeSizeMB=0.8251953125)),segFN=segments_1,generation=1}
   [junit4]   2> 1302258 T3660 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1302259 T3660 oass.SolrIndexSearcher.<init> Opening Searcher@3faaff7b[collection1] main
   [junit4]   2> 1302259 T3660 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 1302260 T3660 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1302260 T3660 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1302261 T3660 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1302261 T3660 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1302261 T3660 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1302262 T3660 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1302262 T3660 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1302262 T3660 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1302263 T3660 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1302263 T3661 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@3faaff7b[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1302264 T3660 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1302265 T3664 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:18962/c_/n collection:control_collection shard:shard1
   [junit4]   2> 1302265 T3630 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 1302265 T3630 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1302266 T3664 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 1302267 T3630 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1302267 T3630 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1302269 T3668 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3982846c name:ZooKeeperConnection Watcher:127.0.0.1:18957/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1302270 T3630 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1302270 T3630 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1302271 T3630 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1302271 T3664 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 1302273 T3656 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1302273 T3664 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1302273 T3664 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 1302274 T3657 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 1302274 T3630 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> ASYNC  NEW_CORE C515 name=collection1 org.apache.solr.core.SolrCore@3622d65f url=http://127.0.0.1:18962/c_/n/collection1 node=127.0.0.1:18962_c_%2Fn C515_STATE=coll:control_collection core:collection1 props:{node_name=127.0.0.1:18962_c_%2Fn, base_url=http://127.0.0.1:18962/c_/n, core=collection1, state=down}
   [junit4]   2> 1302274 T3664 C515 P18962 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:18962/c_/n/collection1/
   [junit4]   2> 1302274 T3630 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 1302275 T3664 C515 P18962 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1302275 T3668 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1302275 T3664 C515 P18962 oasc.SyncStrategy.syncToMe http://127.0.0.1:18962/c_/n/collection1/ has no replicas
   [junit4]   2> 1302275 T3656 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1302275 T3630 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1302276 T3664 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:18962/c_/n/collection1/ shard1
   [junit4]   2> 1302277 T3664 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 1302277 T3630 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1302279 T3671 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@500b8d5c name:ZooKeeperConnection Watcher:127.0.0.1:18957/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1302279 T3630 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1302280 T3630 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1302282 T3656 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1302283 T3657 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 2, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 1302283 T3657 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 1302286 T3657 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:18962/c_/n",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 1302286 T3657 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 1302288 T3656 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1302288 T3668 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1302290 T3656 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1302291 T3668 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1302291 T3656 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1302333 T3664 oasc.ZkController.register We are http://127.0.0.1:18962/c_/n/collection1/ and leader is http://127.0.0.1:18962/c_/n/collection1/
   [junit4]   2> 1302334 T3664 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:18962/c_/n
   [junit4]   2> 1302334 T3664 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1302334 T3664 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 1302334 T3664 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1302336 T3656 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1302337 T3657 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:18962_c_%2Fn",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:18962/c_/n",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1"} current state version: 4
   [junit4]   2> 1302337 T3657 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:18962_c_%2Fn",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:18962/c_/n",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1302441 T3656 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1302441 T3668 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 1304032 T3630 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-1-001/cores/collection1
   [junit4]   2> 1304035 T3630 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-1-001
   [junit4]   2> 1304036 T3630 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1304039 T3630 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:18972
   [junit4]   2> 1304039 T3630 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-1-001/cores, hostContext=/c_/n, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/tempDir-001/jetty1, hostPort=18971, solrconfig=solrconfig.xml}
   [junit4]   2> 1304040 T3630 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 1304040 T3630 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-1-001/'
   [junit4]   2> 1304087 T3630 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-1-001/solr.xml
   [junit4]   2> 1304141 T3630 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-1-001/cores
   [junit4]   2> 1304142 T3630 oasc.CoreContainer.<init> New CoreContainer 527992003
   [junit4]   2> 1304142 T3630 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-1-001/]
   [junit4]   2> 1304143 T3630 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-1-001/lib
   [junit4]   2> 1304143 T3630 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-1-001/lib).
   [junit4]   2> 1304157 T3630 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 1304157 T3630 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 1304157 T3630 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 1304158 T3630 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 1304158 T3630 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 1304158 T3630 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 1304159 T3630 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 1304159 T3630 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 1304159 T3630 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 1304160 T3630 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 1304160 T3630 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 1304160 T3630 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 1304161 T3630 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 1304161 T3630 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 1304162 T3630 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 1304162 T3630 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:18957/solr
   [junit4]   2> 1304162 T3630 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 1304163 T3630 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 1304163 T3630 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1304165 T3683 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@76c4f53f name:ZooKeeperConnection Watcher:127.0.0.1:18957 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1304166 T3630 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1304167 T3630 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 1304169 T3630 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 1304170 T3686 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4e5c907d name:ZooKeeperConnection Watcher:127.0.0.1:18957/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 1304170 T3630 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 1304175 T3630 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 1305180 T3630 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:18971_c_%2Fn
   [junit4]   2> 1305181 T3630 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:18971_c_%2Fn
   [junit4]   2> 1305184 T3630 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 1305186 T3630 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-1-001/cores
   [junit4]   1> CORE DESCRIPTOR: {collection=collection1, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-1-001/cores/collection1/, dataDir=data/, schema=schema.xml, transient=false, coreNodeName=, loadOnStartup=true, config=solrconfig.xml, shard=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-1-001/cores/collection1}
   [junit4]   2> 1305188 T3630 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-1-001/cores/collection1/
   [junit4]   2> 1305188 T3630 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 1305190 T3687 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 1305190 T3687 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1305191 T3656 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1305192 T3687 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 1305192 T3657 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:18971_c_%2Fn",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:18971/c_/n",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down"} current state version: 5
   [junit4]   2> 1305192 T3687 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 1305193 T3657 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:18971_c_%2Fn",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:18971/c_/n",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 1305193 T3687 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 1305193 T3657 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 1305193 T3657 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 1305194 T3687 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 1305194 T3687 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 1305194 T3657 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 1305195 T3686 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1305195 T3686 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 1306194 T3687 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 1306195 T3687 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 1306196 T3687 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 1306196 T3687 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1306197 T3687 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1306197 T3687 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-1-001/cores/collection1/'
   [junit4]   2> 1306216 T3687 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 1306218 T3687 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 1306230 T3687 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 1306256 T3687 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 1306312 T3687 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 1306414 T3687 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 1306426 T3687 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 1306858 T3687 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 1306877 T3687 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 1306880 T3687 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 1306892 T3687 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1306897 T3687 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 1306902 T3687 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1306903 T3687 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1306903 T3687 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1306903 T3687 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 1306904 T3687 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 1306904 T3687 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 1306905 T3687 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 1306905 T3687 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 1306905 T3687 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 1306906 T3687 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@7280b7aa
   [junit4]   2> 1306908 T3687 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-1-001/cores/collection1/data
   [junit4]   2> 1306908 T3687 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 1306909 T3687 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 1306909 T3687 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 1306909 T3687 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=17, maxMergeAtOnceExplicit=12, maxMergedSegmentMB=15.3505859375, floorSegmentMB=0.5556640625, forceMergeDeletesPctAllowed=27.798622184995335, segmentsPerTier=41.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3455216985199636
   [junit4]   2> 1306910 T3687 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@106f5f59 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7fdcd47c; maxCacheMB=0.115234375 maxMergeSizeMB=0.8251953125)),segFN=segments_1,generation=1}
   [junit4]   2> 1306911 T3687 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1306917 T3687 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 1306917 T3687 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 1306918 T3687 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 1306918 T3687 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1306918 T3687 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 1306918 T3687 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 1306919 T3687 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1306919 T3687 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 1306920 T3687 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 1306942 T3687 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1306945 T3687 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1306948 T3687 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1306950 T3687 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 1306958 T3687 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /replication,/admin/plugins,/admin/ping,/update/json/docs,/admin/mbeans,/schema,/update/json,/admin/system,/admin/logging,/update,/admin/threads,/admin/segments,standard,/admin/properties,/admin/luke,/get,/admin/file,/update/csv,/config
   [junit4]   2> 1306960 T3687 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 1306962 T3687 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 1306963 T3687 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 1306964 T3687 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 1306964 T3687 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 1306966 T3687 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1205303418, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 1306967 T3687 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@106f5f59 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@7fdcd47c; maxCacheMB=0.115234375 maxMergeSizeMB=0.8251953125)),segFN=segments_1,generation=1}
   [junit4]   2> 1306967 T3687 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 1306968 T3687 oass.SolrIndexSearcher.<init> Opening Searcher@6d474c3e[collection1] main
   [junit4]   2> 1306968 T3687 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 1306969 T3687 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 1306969 T3687 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 1306970 T3687 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 1306970 T3687 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 1306970 T3687 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1306971 T3687 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 1306971 T3687 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 1306972 T3687 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 1306972 T3687 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 1306973 T3688 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6d474c3e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 1306973 T3687 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 1306974 T3691 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:18971/c_/n collection:collection1 shard:shard3
   [junit4]   2> 1306974 T3630 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 1306975 T3630 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 1306975 T3691 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard3/election
   [junit4]   2> 1306980 T3691 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard3
   [junit4]   2> 1306982 T3656 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1306982 T3691 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 1306982 T3691 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C516 name=collection1 org.apache.solr.core.SolrCore@56bd288a url=http://127.0.0.1:18971/c_/n/collection1 node=127.0.0.1:18971_c_%2Fn C516_STATE=coll:collection1 core:collection1 props:{node_name=127.0.0.1:18971_c_%2Fn, base_url=http://127.0.0.1:18971/c_/n, core=collection1, state=down}
   [junit4]   2> 1306982 T3691 C516 P18971 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:18971/c_/n/collection1/
   [junit4]   2> 1306983 T3657 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 1306983 T3691 C516 P18971 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 1306983 T3691 C516 P18971 oasc.SyncStrategy.syncToMe http://127.0.0.1:18971/c_/n/collection1/ has no replicas
   [junit4]   2> 1306983 T3691 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:18971/c_/n/collection1/ shard3
   [junit4]   2> 1306984 T3691 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard3
   [junit4]   2> 1306984 T3657 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 1
   [junit4]   2> 1306985 T3686 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1306986 T3686 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 2 
   [junit4]   2> 1306988 T3657 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:18971/c_/n",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 1306989 T3657 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 2
   [junit4]   2> 1306990 T3686 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1306991 T3686 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 1306992 T3656 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1307040 T3691 oasc.ZkController.register We are http://127.0.0.1:18971/c_/n/collection1/ and leader is http://127.0.0.1:18971/c_/n/collection1/
   [junit4]   2> 1307040 T3691 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:18971/c_/n
   [junit4]   2> 1307040 T3691 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 1307040 T3691 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 1307040 T3691 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 1307042 T3656 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 1307043 T3657 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:18971_c_%2Fn",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:18971/c_/n",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1"} current state version: 5
   [junit4]   2> 1307043 T3657 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:18971_c_%2Fn",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "shard":"shard3",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:18971/c_/n",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "core_node_name":"core_node1"}
   [junit4]   2> 1307045 T3657 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 3
   [junit4]   2> 1307045 T3691 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 3 
   [junit4]   2> 1307045 T3686 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 1307046 T3686 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 4 
   [junit4]   2> 1308602 T3630 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-2-001/cores/collection1
   [junit4]   2> 1308605 T3630 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-2-001
   [junit4]   2> 1308606 T3630 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 1308609 T3630 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:18980
   [junit4]   2> 1308609 T3630 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/c_/n, hostPort=18979, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-2-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/tempDir-001/jetty2, solrconfig=solrconfig.xml}
   [junit4]   2> 1308610 T3630 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 1308610 T3630 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-2-001/'
   [junit4]   2> 1308653 T3630 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-2-001/solr.xml
   [junit4]   2> 1308703 T3630 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-2-001/cores
   [junit4]   2> 1308704 T3630 oasc.CoreContainer.<init> New CoreContainer 1056895383
   [junit4]   2> 1308704 T3630 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-2-001/]
   [junit4]   2> 1308704 T3630 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 8381C26FCFFDE0A1-001/shard-2-001/lib
   [junit4]   2> 1308705 T3630 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/works

[...truncated too long message...]

rSoftCommitTest 8381C26FCFFDE0A1-001/shard-2-001/cores/collection1/data/index.20150317121107095
   [junit4]   2> 2378531 T38528 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 2378669 T38528 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:38866 38866
   [junit4]   2> 2379770 T38529 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:38866 38866
   [junit4]   2> 2379771 T38529 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/aliases.json
   [junit4]   2> 		3	/solr/configs/conf1
   [junit4]   2> 		2	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		4	/solr/live_nodes
   [junit4]   2> 		3	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 2379772 T38528 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:38884/
   [junit4]   2> 2379772 T38528 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:38872/
   [junit4]   2> 2379772 T38528 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:38903/
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=RecoveryAfterSoftCommitTest -Dtests.method=test -Dtests.seed=8381C26FCFFDE0A1 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=lt -Dtests.timezone=America/Boa_Vista -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   63.7s J3 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([8381C26FCFFDE0A1:BD5FDB561018D59]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 2379801 T38528 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 8381C26FCFFDE0A1-001
   [junit4]   2> NOTE: test params are: codec=Lucene50, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=lt, timezone=America/Boa_Vista
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=157499272,total=382205952
   [junit4]   2> NOTE: All tests run in this JVM: [TestBulkSchemaAPI, TestUtils, TestDynamicFieldResource, ClusterStateUpdateTest, IgnoreCommitOptimizeUpdateProcessorFactoryTest, RequestLoggingTest, DistributedMLTComponentTest, SharedFSAutoReplicaFailoverTest, CurrencyFieldXmlFileTest, TestDistributedGrouping, UUIDUpdateProcessorFallbackTest, TestIndexSearcher, AsyncMigrateRouteKeyTest, DistribDocExpirationUpdateProcessorTest, TestJmxMonitoredMap, TestPostingsSolrHighlighter, DocValuesMultiTest, SuggesterWFSTTest, TestStressUserVersions, TestFiltering, DeleteShardTest, TestPseudoReturnFields, QueryResultKeyTest, TestSolrConfigHandlerConcurrent, DocValuesTest, UpdateRequestProcessorFactoryTest, DistributedFacetPivotLargeTest, TestWordDelimiterFilterFactory, TestDefaultSimilarityFactory, CSVRequestHandlerTest, TestSchemaManager, TestIntervalFaceting, HdfsUnloadDistributedZkTest, DefaultValueUpdateProcessorTest, FacetPivotSmallTest, ExternalCollectionsTest, MultiThreadedOCPTest, AnalysisAfterCoreReloadTest, TestValueSourceCache, TestArbitraryIndexDir, DistributedQueryComponentOptimizationTest, TestLRUStatsCache, JsonLoaderTest, TestClassNameShortening, TestManagedSchema, TestSystemIdResolver, TestPivotHelperCode, UnloadDistributedZkTest, LeaderElectionTest, DateMathParserTest, TestSolrQueryParserDefaultOperatorResource, TestDefaultSearchFieldResource, MultiTermTest, TestCloudManagedSchema, OutOfBoxZkACLAndCredentialsProvidersTest, AlternateDirectoryTest, SyncSliceTest, LeaderElectionIntegrationTest, ShardRoutingTest, ZkSolrClientTest, TestZkChroot, TestRandomDVFaceting, ZkCLITest, TestDistributedSearch, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J3 in 63.76s, 1 test, 1 error <<< FAILURES!

[...truncated 285 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 474 suites, 1889 tests, 3 errors, 58 ignored (9 assumptions)

Total time: 161 minutes 48 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 751655395 bytes
Compression is 0.0%
Took 2 min 15 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 788 - Still Failing

Posted by Michael McCandless <lu...@mikemccandless.com>.
I committed a fix ... MockRandomMP was failing to take IW's buffered
deletes into account, so it though the index was in fact optimized.

Mike McCandless

http://blog.mikemccandless.com


On Mon, Mar 16, 2015 at 10:13 AM, Robert Muir <rc...@gmail.com> wrote:
> It seems the test assumes after it does forceMerge(1), that it can
> open a reader that has no deletes. but this is not the case.
>
> with its mockrandom mergepolicy, it ends out with a single-segment
> index like this:
> leaf: _ak(5.1.0):c5727/988:delGen=1 maxDoc=5727, deleted=988
>
> On Mon, Mar 16, 2015 at 9:38 AM, Michael McCandless
> <lu...@mikemccandless.com> wrote:
>> Reproduces ... I'll dig.
>>
>> Mike McCandless
>>
>> http://blog.mikemccandless.com
>>
>>
>> On Mon, Mar 16, 2015 at 8:40 AM, Apache Jenkins Server
>> <je...@builds.apache.org> wrote:
>>> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/788/
>>>
>>> 1 tests failed.
>>> REGRESSION:  org.apache.lucene.codecs.lucene49.TestLucene49NormsFormat.testUndeadNorms
>>>
>>> Error Message:
>>> expected:<0> but was:<121>
>>>
>>> Stack Trace:
>>> java.lang.AssertionError: expected:<0> but was:<121>
>>>         at __randomizedtesting.SeedInfo.seed([A05EC27B0D94C2A8:AD8B8616D87AD314]:0)
>>>         at org.junit.Assert.fail(Assert.java:93)
>>>         at org.junit.Assert.failNotEquals(Assert.java:647)
>>>         at org.junit.Assert.assertEquals(Assert.java:128)
>>>         at org.junit.Assert.assertEquals(Assert.java:472)
>>>         at org.junit.Assert.assertEquals(Assert.java:456)
>>>         at org.apache.lucene.index.BaseNormsFormatTestCase.testUndeadNorms(BaseNormsFormatTestCase.java:397)
>>>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>>>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>>>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>>         at java.lang.reflect.Method.invoke(Method.java:606)
>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
>>>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>>>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>>>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>>>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>>>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
>>>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>>>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>>>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>>>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>>>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>>>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>>>         at java.lang.Thread.run(Thread.java:745)
>>>
>>>
>>>
>>>
>>> Build Log:
>>> [...truncated 4911 lines...]
>>>    [junit4] Suite: org.apache.lucene.codecs.lucene49.TestLucene49NormsFormat
>>>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>>>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene49NormsFormat -Dtests.method=testUndeadNorms -Dtests.seed=A05EC27B0D94C2A8 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=pl -Dtests.timezone=Europe/Riga -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
>>>    [junit4] FAILURE 1.85s J1 | TestLucene49NormsFormat.testUndeadNorms <<<
>>>    [junit4]    > Throwable #1: java.lang.AssertionError: expected:<0> but was:<121>
>>>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([A05EC27B0D94C2A8:AD8B8616D87AD314]:0)
>>>    [junit4]    >        at org.apache.lucene.index.BaseNormsFormatTestCase.testUndeadNorms(BaseNormsFormatTestCase.java:397)
>>>    [junit4]    >        at java.lang.Thread.run(Thread.java:745)
>>>    [junit4] IGNOR/A 0.01s J1 | TestLucene49NormsFormat.testMergeStability
>>>    [junit4]    > Assumption #1: The MockRandom PF randomizes content on the fly, so we can't check it
>>>    [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/backward-codecs/test/J1/temp/lucene.codecs.lucene49.TestLucene49NormsFormat A05EC27B0D94C2A8-001
>>>    [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50), sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {content=DFR GB1, foobar=DFR G3(800.0)}, locale=pl, timezone=Europe/Riga
>>>    [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=311078728,total=472383488
>>>    [junit4]   2> NOTE: All tests run in this JVM: [TestDocValuesUpdatesOnOldSegments, TestLucene42NormsFormat, TestLucene40BlockFormat, TestBackwardsCompatibility, TestBitVector, TestLucene40StoredFieldsFormat, TestLucene41PostingsFormat3, TestLucene41StoredFieldsFormat, TestLucene49NormsFormat]
>>>    [junit4] Completed on J1 in 262.45s, 17 tests, 1 failure, 1 skipped <<< FAILURES!
>>>
>>> [...truncated 27 lines...]
>>> BUILD FAILED
>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:456: The following error occurred while executing this line:
>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:2166: The following error occurred while executing this line:
>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/module-build.xml:58: The following error occurred while executing this line:
>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 29 suites, 689 tests, 1 failure, 74 ignored (74 assumptions)
>>>
>>> Total time: 94 minutes 49 seconds
>>> Build step 'Invoke Ant' marked build as failure
>>> Archiving artifacts
>>> Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
>>> Archived 5 artifacts
>>> Archive block size is 32768
>>> Received 0 blocks and 1412631 bytes
>>> Compression is 0.0%
>>> Took 0.65 sec
>>> Recording test results
>>> Email was triggered for: Failure
>>> Sending email for trigger: Failure
>>>
>>>
>>>
>>>
>>> ---------------------------------------------------------------------
>>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>>> For additional commands, e-mail: dev-help@lucene.apache.org
>>
>> ---------------------------------------------------------------------
>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>> For additional commands, e-mail: dev-help@lucene.apache.org
>>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


Re: [JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 788 - Still Failing

Posted by Robert Muir <rc...@gmail.com>.
It seems the test assumes after it does forceMerge(1), that it can
open a reader that has no deletes. but this is not the case.

with its mockrandom mergepolicy, it ends out with a single-segment
index like this:
leaf: _ak(5.1.0):c5727/988:delGen=1 maxDoc=5727, deleted=988

On Mon, Mar 16, 2015 at 9:38 AM, Michael McCandless
<lu...@mikemccandless.com> wrote:
> Reproduces ... I'll dig.
>
> Mike McCandless
>
> http://blog.mikemccandless.com
>
>
> On Mon, Mar 16, 2015 at 8:40 AM, Apache Jenkins Server
> <je...@builds.apache.org> wrote:
>> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/788/
>>
>> 1 tests failed.
>> REGRESSION:  org.apache.lucene.codecs.lucene49.TestLucene49NormsFormat.testUndeadNorms
>>
>> Error Message:
>> expected:<0> but was:<121>
>>
>> Stack Trace:
>> java.lang.AssertionError: expected:<0> but was:<121>
>>         at __randomizedtesting.SeedInfo.seed([A05EC27B0D94C2A8:AD8B8616D87AD314]:0)
>>         at org.junit.Assert.fail(Assert.java:93)
>>         at org.junit.Assert.failNotEquals(Assert.java:647)
>>         at org.junit.Assert.assertEquals(Assert.java:128)
>>         at org.junit.Assert.assertEquals(Assert.java:472)
>>         at org.junit.Assert.assertEquals(Assert.java:456)
>>         at org.apache.lucene.index.BaseNormsFormatTestCase.testUndeadNorms(BaseNormsFormatTestCase.java:397)
>>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>         at java.lang.reflect.Method.invoke(Method.java:606)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
>>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
>>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>>         at java.lang.Thread.run(Thread.java:745)
>>
>>
>>
>>
>> Build Log:
>> [...truncated 4911 lines...]
>>    [junit4] Suite: org.apache.lucene.codecs.lucene49.TestLucene49NormsFormat
>>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene49NormsFormat -Dtests.method=testUndeadNorms -Dtests.seed=A05EC27B0D94C2A8 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=pl -Dtests.timezone=Europe/Riga -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
>>    [junit4] FAILURE 1.85s J1 | TestLucene49NormsFormat.testUndeadNorms <<<
>>    [junit4]    > Throwable #1: java.lang.AssertionError: expected:<0> but was:<121>
>>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([A05EC27B0D94C2A8:AD8B8616D87AD314]:0)
>>    [junit4]    >        at org.apache.lucene.index.BaseNormsFormatTestCase.testUndeadNorms(BaseNormsFormatTestCase.java:397)
>>    [junit4]    >        at java.lang.Thread.run(Thread.java:745)
>>    [junit4] IGNOR/A 0.01s J1 | TestLucene49NormsFormat.testMergeStability
>>    [junit4]    > Assumption #1: The MockRandom PF randomizes content on the fly, so we can't check it
>>    [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/backward-codecs/test/J1/temp/lucene.codecs.lucene49.TestLucene49NormsFormat A05EC27B0D94C2A8-001
>>    [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50), sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {content=DFR GB1, foobar=DFR G3(800.0)}, locale=pl, timezone=Europe/Riga
>>    [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=311078728,total=472383488
>>    [junit4]   2> NOTE: All tests run in this JVM: [TestDocValuesUpdatesOnOldSegments, TestLucene42NormsFormat, TestLucene40BlockFormat, TestBackwardsCompatibility, TestBitVector, TestLucene40StoredFieldsFormat, TestLucene41PostingsFormat3, TestLucene41StoredFieldsFormat, TestLucene49NormsFormat]
>>    [junit4] Completed on J1 in 262.45s, 17 tests, 1 failure, 1 skipped <<< FAILURES!
>>
>> [...truncated 27 lines...]
>> BUILD FAILED
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:456: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:2166: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/module-build.xml:58: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 29 suites, 689 tests, 1 failure, 74 ignored (74 assumptions)
>>
>> Total time: 94 minutes 49 seconds
>> Build step 'Invoke Ant' marked build as failure
>> Archiving artifacts
>> Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
>> Archived 5 artifacts
>> Archive block size is 32768
>> Received 0 blocks and 1412631 bytes
>> Compression is 0.0%
>> Took 0.65 sec
>> Recording test results
>> Email was triggered for: Failure
>> Sending email for trigger: Failure
>>
>>
>>
>>
>> ---------------------------------------------------------------------
>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>> For additional commands, e-mail: dev-help@lucene.apache.org
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


Re: [JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 788 - Still Failing

Posted by Michael McCandless <lu...@mikemccandless.com>.
Reproduces ... I'll dig.

Mike McCandless

http://blog.mikemccandless.com


On Mon, Mar 16, 2015 at 8:40 AM, Apache Jenkins Server
<je...@builds.apache.org> wrote:
> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/788/
>
> 1 tests failed.
> REGRESSION:  org.apache.lucene.codecs.lucene49.TestLucene49NormsFormat.testUndeadNorms
>
> Error Message:
> expected:<0> but was:<121>
>
> Stack Trace:
> java.lang.AssertionError: expected:<0> but was:<121>
>         at __randomizedtesting.SeedInfo.seed([A05EC27B0D94C2A8:AD8B8616D87AD314]:0)
>         at org.junit.Assert.fail(Assert.java:93)
>         at org.junit.Assert.failNotEquals(Assert.java:647)
>         at org.junit.Assert.assertEquals(Assert.java:128)
>         at org.junit.Assert.assertEquals(Assert.java:472)
>         at org.junit.Assert.assertEquals(Assert.java:456)
>         at org.apache.lucene.index.BaseNormsFormatTestCase.testUndeadNorms(BaseNormsFormatTestCase.java:397)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
>
>
> Build Log:
> [...truncated 4911 lines...]
>    [junit4] Suite: org.apache.lucene.codecs.lucene49.TestLucene49NormsFormat
>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene49NormsFormat -Dtests.method=testUndeadNorms -Dtests.seed=A05EC27B0D94C2A8 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=pl -Dtests.timezone=Europe/Riga -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
>    [junit4] FAILURE 1.85s J1 | TestLucene49NormsFormat.testUndeadNorms <<<
>    [junit4]    > Throwable #1: java.lang.AssertionError: expected:<0> but was:<121>
>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([A05EC27B0D94C2A8:AD8B8616D87AD314]:0)
>    [junit4]    >        at org.apache.lucene.index.BaseNormsFormatTestCase.testUndeadNorms(BaseNormsFormatTestCase.java:397)
>    [junit4]    >        at java.lang.Thread.run(Thread.java:745)
>    [junit4] IGNOR/A 0.01s J1 | TestLucene49NormsFormat.testMergeStability
>    [junit4]    > Assumption #1: The MockRandom PF randomizes content on the fly, so we can't check it
>    [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/backward-codecs/test/J1/temp/lucene.codecs.lucene49.TestLucene49NormsFormat A05EC27B0D94C2A8-001
>    [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50), sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {content=DFR GB1, foobar=DFR G3(800.0)}, locale=pl, timezone=Europe/Riga
>    [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=311078728,total=472383488
>    [junit4]   2> NOTE: All tests run in this JVM: [TestDocValuesUpdatesOnOldSegments, TestLucene42NormsFormat, TestLucene40BlockFormat, TestBackwardsCompatibility, TestBitVector, TestLucene40StoredFieldsFormat, TestLucene41PostingsFormat3, TestLucene41StoredFieldsFormat, TestLucene49NormsFormat]
>    [junit4] Completed on J1 in 262.45s, 17 tests, 1 failure, 1 skipped <<< FAILURES!
>
> [...truncated 27 lines...]
> BUILD FAILED
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:456: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:2166: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/module-build.xml:58: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 29 suites, 689 tests, 1 failure, 74 ignored (74 assumptions)
>
> Total time: 94 minutes 49 seconds
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
> Archived 5 artifacts
> Archive block size is 32768
> Received 0 blocks and 1412631 bytes
> Compression is 0.0%
> Took 0.65 sec
> Recording test results
> Email was triggered for: Failure
> Sending email for trigger: Failure
>
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 788 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/788/

1 tests failed.
REGRESSION:  org.apache.lucene.codecs.lucene49.TestLucene49NormsFormat.testUndeadNorms

Error Message:
expected:<0> but was:<121>

Stack Trace:
java.lang.AssertionError: expected:<0> but was:<121>
	at __randomizedtesting.SeedInfo.seed([A05EC27B0D94C2A8:AD8B8616D87AD314]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:472)
	at org.junit.Assert.assertEquals(Assert.java:456)
	at org.apache.lucene.index.BaseNormsFormatTestCase.testUndeadNorms(BaseNormsFormatTestCase.java:397)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 4911 lines...]
   [junit4] Suite: org.apache.lucene.codecs.lucene49.TestLucene49NormsFormat
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLucene49NormsFormat -Dtests.method=testUndeadNorms -Dtests.seed=A05EC27B0D94C2A8 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=pl -Dtests.timezone=Europe/Riga -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 1.85s J1 | TestLucene49NormsFormat.testUndeadNorms <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: expected:<0> but was:<121>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([A05EC27B0D94C2A8:AD8B8616D87AD314]:0)
   [junit4]    > 	at org.apache.lucene.index.BaseNormsFormatTestCase.testUndeadNorms(BaseNormsFormatTestCase.java:397)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4] IGNOR/A 0.01s J1 | TestLucene49NormsFormat.testMergeStability
   [junit4]    > Assumption #1: The MockRandom PF randomizes content on the fly, so we can't check it
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/backward-codecs/test/J1/temp/lucene.codecs.lucene49.TestLucene49NormsFormat A05EC27B0D94C2A8-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50), sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {content=DFR GB1, foobar=DFR G3(800.0)}, locale=pl, timezone=Europe/Riga
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=311078728,total=472383488
   [junit4]   2> NOTE: All tests run in this JVM: [TestDocValuesUpdatesOnOldSegments, TestLucene42NormsFormat, TestLucene40BlockFormat, TestBackwardsCompatibility, TestBitVector, TestLucene40StoredFieldsFormat, TestLucene41PostingsFormat3, TestLucene41StoredFieldsFormat, TestLucene49NormsFormat]
   [junit4] Completed on J1 in 262.45s, 17 tests, 1 failure, 1 skipped <<< FAILURES!

[...truncated 27 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:456: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:2166: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/module-build.xml:58: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 29 suites, 689 tests, 1 failure, 74 ignored (74 assumptions)

Total time: 94 minutes 49 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 1412631 bytes
Compression is 0.0%
Took 0.65 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 787 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/787/

4 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:38317/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38317/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([244FE66A6BD041BA:AC1BD9B0C52C2C42]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22082/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22082/collection1
	at __randomizedtesting.SeedInfo.seed([244FE66A6BD041BA:AC1BD9B0C52C2C42]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22082/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48314/hg/h/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48314/hg/h/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([244FE66A6BD041BA:AC1BD9B0C52C2C42]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:48314/hg/h/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 53 more


FAILED:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1426450760315,generation=2,filelist=[_1bk.fdt, _1bk.fdx, _1bk.fnm, _1bk.nvd, _1bk.nvm, _1bk.si, _1bk_BlockTreeOrds_0.doc, _1bk_BlockTreeOrds_0.tio, _1bk_BlockTreeOrds_0.tipo, _1bx.fdt, _1bx.fdx, _1bx.fnm, _1bx.nvd, _1bx.nvm, _1bx.si, _1bx_BlockTreeOrds_0.doc, _1bx_BlockTreeOrds_0.tio, _1bx_BlockTreeOrds_0.tipo, _1c0.fdt, _1c0.fdx, _1c0.fnm, _1c0.nvd, _1c0.nvm, _1c0.si, _1c0_BlockTreeOrds_0.doc, _1c0_BlockTreeOrds_0.tio, _1c0_BlockTreeOrds_0.tipo, _1c4.cfe, _1c4.cfs, _1c4.si, _1c5.fdt, _1c5.fdx, _1c5.fnm, _1c5.nvd, _1c5.nvm, _1c5.si, _1c5_BlockTreeOrds_0.doc, _1c5_BlockTreeOrds_0.tio, _1c5_BlockTreeOrds_0.tipo, _1ca.fdt, _1ca.fdx, _1ca.fnm, _1ca.nvd, _1ca.nvm, _1ca.si, _1ca_BlockTreeOrds_0.doc, _1ca_BlockTreeOrds_0.tio, _1ca_BlockTreeOrds_0.tipo, _1cc.cfe, _1cc.cfs, _1cc.si, _1cd.fdt, _1cd.fdx, _1cd.fnm, _1cd.nvd, _1cd.nvm, _1cd.si, _1cd_BlockTreeOrds_0.doc, _1cd_BlockTreeOrds_0.tio, _1cd_BlockTreeOrds_0.tipo, segments_2]}]> but was:<[{indexVersion=1426450760315,generation=2,filelist=[_1bk.fdt, _1bk.fdx, _1bk.fnm, _1bk.nvd, _1bk.nvm, _1bk.si, _1bk_BlockTreeOrds_0.doc, _1bk_BlockTreeOrds_0.tio, _1bk_BlockTreeOrds_0.tipo, _1bx.fdt, _1bx.fdx, _1bx.fnm, _1bx.nvd, _1bx.nvm, _1bx.si, _1bx_BlockTreeOrds_0.doc, _1bx_BlockTreeOrds_0.tio, _1bx_BlockTreeOrds_0.tipo, _1c0.fdt, _1c0.fdx, _1c0.fnm, _1c0.nvd, _1c0.nvm, _1c0.si, _1c0_BlockTreeOrds_0.doc, _1c0_BlockTreeOrds_0.tio, _1c0_BlockTreeOrds_0.tipo, _1c4.cfe, _1c4.cfs, _1c4.si, _1c5.fdt, _1c5.fdx, _1c5.fnm, _1c5.nvd, _1c5.nvm, _1c5.si, _1c5_BlockTreeOrds_0.doc, _1c5_BlockTreeOrds_0.tio, _1c5_BlockTreeOrds_0.tipo, _1ca.fdt, _1ca.fdx, _1ca.fnm, _1ca.nvd, _1ca.nvm, _1ca.si, _1ca_BlockTreeOrds_0.doc, _1ca_BlockTreeOrds_0.tio, _1ca_BlockTreeOrds_0.tipo, _1cc.cfe, _1cc.cfs, _1cc.si, _1cd.fdt, _1cd.fdx, _1cd.fnm, _1cd.nvd, _1cd.nvm, _1cd.si, _1cd_BlockTreeOrds_0.doc, _1cd_BlockTreeOrds_0.tio, _1cd_BlockTreeOrds_0.tipo, segments_2]}, {indexVersion=1426450760315,generation=3,filelist=[_1ca.fdt, _1ca.fdx, _1ca.fnm, _1ca.nvd, _1ca.nvm, _1ca.si, _1ca_BlockTreeOrds_0.doc, _1ca_BlockTreeOrds_0.tio, _1ca_BlockTreeOrds_0.tipo, _1cc.cfe, _1cc.cfs, _1cc.si, _1cd.fdt, _1cd.fdx, _1cd.fnm, _1cd.nvd, _1cd.nvm, _1cd.si, _1cd_BlockTreeOrds_0.doc, _1cd_BlockTreeOrds_0.tio, _1cd_BlockTreeOrds_0.tipo, _1ce.cfe, _1ce.cfs, _1ce.si, segments_3]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1426450760315,generation=2,filelist=[_1bk.fdt, _1bk.fdx, _1bk.fnm, _1bk.nvd, _1bk.nvm, _1bk.si, _1bk_BlockTreeOrds_0.doc, _1bk_BlockTreeOrds_0.tio, _1bk_BlockTreeOrds_0.tipo, _1bx.fdt, _1bx.fdx, _1bx.fnm, _1bx.nvd, _1bx.nvm, _1bx.si, _1bx_BlockTreeOrds_0.doc, _1bx_BlockTreeOrds_0.tio, _1bx_BlockTreeOrds_0.tipo, _1c0.fdt, _1c0.fdx, _1c0.fnm, _1c0.nvd, _1c0.nvm, _1c0.si, _1c0_BlockTreeOrds_0.doc, _1c0_BlockTreeOrds_0.tio, _1c0_BlockTreeOrds_0.tipo, _1c4.cfe, _1c4.cfs, _1c4.si, _1c5.fdt, _1c5.fdx, _1c5.fnm, _1c5.nvd, _1c5.nvm, _1c5.si, _1c5_BlockTreeOrds_0.doc, _1c5_BlockTreeOrds_0.tio, _1c5_BlockTreeOrds_0.tipo, _1ca.fdt, _1ca.fdx, _1ca.fnm, _1ca.nvd, _1ca.nvm, _1ca.si, _1ca_BlockTreeOrds_0.doc, _1ca_BlockTreeOrds_0.tio, _1ca_BlockTreeOrds_0.tipo, _1cc.cfe, _1cc.cfs, _1cc.si, _1cd.fdt, _1cd.fdx, _1cd.fnm, _1cd.nvd, _1cd.nvm, _1cd.si, _1cd_BlockTreeOrds_0.doc, _1cd_BlockTreeOrds_0.tio, _1cd_BlockTreeOrds_0.tipo, segments_2]}]> but was:<[{indexVersion=1426450760315,generation=2,filelist=[_1bk.fdt, _1bk.fdx, _1bk.fnm, _1bk.nvd, _1bk.nvm, _1bk.si, _1bk_BlockTreeOrds_0.doc, _1bk_BlockTreeOrds_0.tio, _1bk_BlockTreeOrds_0.tipo, _1bx.fdt, _1bx.fdx, _1bx.fnm, _1bx.nvd, _1bx.nvm, _1bx.si, _1bx_BlockTreeOrds_0.doc, _1bx_BlockTreeOrds_0.tio, _1bx_BlockTreeOrds_0.tipo, _1c0.fdt, _1c0.fdx, _1c0.fnm, _1c0.nvd, _1c0.nvm, _1c0.si, _1c0_BlockTreeOrds_0.doc, _1c0_BlockTreeOrds_0.tio, _1c0_BlockTreeOrds_0.tipo, _1c4.cfe, _1c4.cfs, _1c4.si, _1c5.fdt, _1c5.fdx, _1c5.fnm, _1c5.nvd, _1c5.nvm, _1c5.si, _1c5_BlockTreeOrds_0.doc, _1c5_BlockTreeOrds_0.tio, _1c5_BlockTreeOrds_0.tipo, _1ca.fdt, _1ca.fdx, _1ca.fnm, _1ca.nvd, _1ca.nvm, _1ca.si, _1ca_BlockTreeOrds_0.doc, _1ca_BlockTreeOrds_0.tio, _1ca_BlockTreeOrds_0.tipo, _1cc.cfe, _1cc.cfs, _1cc.si, _1cd.fdt, _1cd.fdx, _1cd.fnm, _1cd.nvd, _1cd.nvm, _1cd.si, _1cd_BlockTreeOrds_0.doc, _1cd_BlockTreeOrds_0.tio, _1cd_BlockTreeOrds_0.tipo, segments_2]}, {indexVersion=1426450760315,generation=3,filelist=[_1ca.fdt, _1ca.fdx, _1ca.fnm, _1ca.nvd, _1ca.nvm, _1ca.si, _1ca_BlockTreeOrds_0.doc, _1ca_BlockTreeOrds_0.tio, _1ca_BlockTreeOrds_0.tipo, _1cc.cfe, _1cc.cfs, _1cc.si, _1cd.fdt, _1cd.fdx, _1cd.fnm, _1cd.nvd, _1cd.nvm, _1cd.si, _1cd_BlockTreeOrds_0.doc, _1cd_BlockTreeOrds_0.tio, _1cd_BlockTreeOrds_0.tipo, _1ce.cfe, _1ce.cfs, _1ce.si, segments_3]}]>
	at __randomizedtesting.SeedInfo.seed([244FE66A6BD041BA:198FD5A1B984FB9]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1137)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 10054 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/init-core-data-001
   [junit4]   2> 434649 T4991 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /hg/h
   [junit4]   2> 434654 T4991 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 434655 T4992 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 434755 T4991 oasc.ZkTestServer.run start zk server on port:48272
   [junit4]   2> 434756 T4991 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 434757 T4991 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 434762 T4999 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@59481d44 name:ZooKeeperConnection Watcher:127.0.0.1:48272 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 434763 T4991 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 434763 T4991 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 434763 T4991 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 434766 T4991 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 434767 T4991 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 434769 T5002 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6085b2ea name:ZooKeeperConnection Watcher:127.0.0.1:48272/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 434769 T4991 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 434769 T4991 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 434770 T4991 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 434772 T4991 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 434773 T4991 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 434774 T4991 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 434776 T4991 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 434776 T4991 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 434779 T4991 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 434779 T4991 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 434781 T4991 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 434782 T4991 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 434783 T4991 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 434784 T4991 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 434785 T4991 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 434786 T4991 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 434787 T4991 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 434788 T4991 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 434789 T4991 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 434790 T4991 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 434791 T4991 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 434792 T4991 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 434793 T4991 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 434793 T4991 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 434795 T4991 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 434796 T4991 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 434797 T4991 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 434798 T4991 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 436284 T4991 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/control-001/cores/collection1
   [junit4]   2> 436289 T4991 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 436293 T4991 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:48285
   [junit4]   2> 436294 T4991 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/control-001/cores, hostContext=/hg/h, hostPort=48284, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/tempDir-001/control/data}
   [junit4]   2> 436294 T4991 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 436295 T4991 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/control-001/'
   [junit4]   2> 436340 T4991 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/control-001/solr.xml
   [junit4]   2> 436396 T4991 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/control-001/cores
   [junit4]   2> 436397 T4991 oasc.CoreContainer.<init> New CoreContainer 783473999
   [junit4]   2> 436397 T4991 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/control-001/]
   [junit4]   2> 436398 T4991 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/control-001/lib
   [junit4]   2> 436399 T4991 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/control-001/lib).
   [junit4]   2> 436412 T4991 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 436413 T4991 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 436413 T4991 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 436414 T4991 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 436414 T4991 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 436415 T4991 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 436415 T4991 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 436416 T4991 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 436416 T4991 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 436416 T4991 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 436417 T4991 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 436418 T4991 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 436418 T4991 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 436418 T4991 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 436419 T4991 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 436419 T4991 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48272/solr
   [junit4]   2> 436420 T4991 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 436420 T4991 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 436421 T4991 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 436423 T5014 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3fc09336 name:ZooKeeperConnection Watcher:127.0.0.1:48272 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 436423 T4991 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 436424 T4991 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 436426 T4991 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 436428 T5017 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1ccdcb16 name:ZooKeeperConnection Watcher:127.0.0.1:48272/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 436428 T4991 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 436430 T4991 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 436432 T4991 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 436434 T4991 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 436436 T4991 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 436438 T4991 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 436440 T4991 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 436441 T4991 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 436442 T4991 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 436443 T4991 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48284_hg%2Fh
   [junit4]   2> 436444 T4991 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:48284_hg%2Fh
   [junit4]   2> 436445 T4991 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 436447 T4991 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 436448 T4991 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 436449 T4991 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:48284_hg%2Fh
   [junit4]   2> 436449 T4991 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 436451 T4991 oasc.Overseer.start Overseer (id=93483792347758595-127.0.0.1:48284_hg%2Fh-n_0000000000) starting
   [junit4]   2> 436453 T4991 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 436458 T4991 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 436458 T5019 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 436458 T4991 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 436459 T5018 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 436463 T4991 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/control-001/cores
   [junit4]   1> CORE DESCRIPTOR: {shard=, config=solrconfig.xml, transient=false, name=collection1, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/control-001/cores/collection1, schema=schema.xml, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/control-001/cores/collection1/, collection=control_collection, coreNodeName=}
   [junit4]   2> 436466 T4991 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/control-001/cores/collection1/
   [junit4]   2> 436466 T4991 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 436467 T5021 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 436468 T5021 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 436468 T5021 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 436468 T5017 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 436469 T5018 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48284/hg/h",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:48284_hg%2Fh",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1"} current state version: 0
   [junit4]   2> 436470 T5018 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48284/hg/h",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:48284_hg%2Fh",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 436470 T5018 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 436471 T5018 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 436472 T5017 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 437469 T5021 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 437470 T5021 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 437470 T5021 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 437471 T5021 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 437471 T5021 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 437472 T5021 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/control-001/cores/collection1/'
   [junit4]   2> 437488 T5021 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 437491 T5021 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 437505 T5021 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 437529 T5021 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 437582 T5021 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 437583 T5021 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 437595 T5021 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 438017 T5021 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 438034 T5021 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 438037 T5021 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 438048 T5021 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 438052 T5021 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 438056 T5021 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 438057 T5021 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 438057 T5021 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 438058 T5021 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 438058 T5021 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 438059 T5021 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 438059 T5021 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 438059 T5021 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 438060 T5021 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/control-001/cores/collection1/data/
   [junit4]   2> 438060 T5021 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5208513
   [junit4]   2> 438061 T5021 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/control-001/cores/collection1/data
   [junit4]   2> 438061 T5021 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/control-001/cores/collection1/data/index/
   [junit4]   2> 438061 T5021 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 438062 T5021 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/control-001/cores/collection1/data/index
   [junit4]   2> 438062 T5021 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=20, maxMergedSegmentMB=25.1201171875, floorSegmentMB=1.1103515625, forceMergeDeletesPctAllowed=27.54578614969428, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 438063 T5021 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6a45b04a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@53900c8),segFN=segments_1,generation=1}
   [junit4]   2> 438063 T5021 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 438070 T5021 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 438071 T5021 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 438071 T5021 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 438071 T5021 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 438071 T5021 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 438072 T5021 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 438072 T5021 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 438072 T5021 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 438073 T5021 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 438092 T5021 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 438094 T5021 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 438097 T5021 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 438099 T5021 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 438106 T5021 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/segments,/admin/mbeans,/admin/system,/admin/ping,/admin/luke,/admin/logging,/admin/plugins,/admin/properties,/admin/file,/update/json,/schema,/get,/config,/update/csv,/admin/threads,/update,/update/json/docs,/replication,standard
   [junit4]   2> 438108 T5021 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 438110 T5021 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 438110 T5021 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 438112 T5021 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 438112 T5021 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 438113 T5021 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=11, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7170157941624185]
   [junit4]   2> 438113 T5021 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@6a45b04a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@53900c8),segFN=segments_1,generation=1}
   [junit4]   2> 438114 T5021 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 438114 T5021 oass.SolrIndexSearcher.<init> Opening Searcher@2736a7e7[collection1] main
   [junit4]   2> 438114 T5021 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 438115 T5021 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 438115 T5021 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 438116 T5021 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 438116 T5021 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 438116 T5021 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 438117 T5021 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 438117 T5021 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 438117 T5021 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 438118 T5021 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 438119 T5022 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2736a7e7[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 438119 T5021 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 438120 T5025 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:48284/hg/h collection:control_collection shard:shard1
   [junit4]   2> 438121 T4991 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 438121 T4991 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 438122 T5025 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 438125 T4991 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 438125 T4991 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 438126 T5025 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 438127 T5029 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6f364526 name:ZooKeeperConnection Watcher:127.0.0.1:48272/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 438127 T4991 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 438128 T5017 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 438128 T4991 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 438128 T5025 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 438128 T4991 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 438128 T5018 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 438128 T5025 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C324 name=collection1 org.apache.solr.core.SolrCore@1f87f19d url=http://127.0.0.1:48284/hg/h/collection1 node=127.0.0.1:48284_hg%2Fh C324_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:48284/hg/h, state=down, node_name=127.0.0.1:48284_hg%2Fh, core=collection1}
   [junit4]   2> 438129 T5025 C324 P48284 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:48284/hg/h/collection1/
   [junit4]   2> 438130 T5025 C324 P48284 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 438130 T5025 C324 P48284 oasc.SyncStrategy.syncToMe http://127.0.0.1:48284/hg/h/collection1/ has no replicas
   [junit4]   2> 438130 T5025 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:48284/hg/h/collection1/ shard1
   [junit4]   2> 438130 T5029 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 438130 T5017 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 438131 T5025 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 438132 T4991 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 438135 T5017 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 438136 T5018 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48284/hg/h",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 438240 T5029 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 438240 T5017 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 438288 T5025 oasc.ZkController.register We are http://127.0.0.1:48284/hg/h/collection1/ and leader is http://127.0.0.1:48284/hg/h/collection1/
   [junit4]   2> 438289 T5025 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:48284/hg/h
   [junit4]   2> 438289 T5025 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 438289 T5025 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 438289 T5025 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 438291 T5017 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 438292 T5018 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48284/hg/h",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:48284_hg%2Fh",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1"} current state version: 3
   [junit4]   2> 438292 T5018 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48284/hg/h",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:48284_hg%2Fh",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 438396 T5017 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 438396 T5029 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 439670 T4991 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/shard-1-001/cores/collection1
   [junit4]   2> 439673 T4991 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/shard-1-001
   [junit4]   2> 439675 T4991 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 439677 T4991 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:48292
   [junit4]   2> 439678 T4991 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/shard-1-001/cores, hostContext=/hg/h, hostPort=48291, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/tempDir-001/jetty1}
   [junit4]   2> 439678 T4991 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 439679 T4991 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/shard-1-001/'
   [junit4]   2> 439728 T4991 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/shard-1-001/solr.xml
   [junit4]   2> 439786 T4991 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/shard-1-001/cores
   [junit4]   2> 439787 T4991 oasc.CoreContainer.<init> New CoreContainer 1392136823
   [junit4]   2> 439787 T4991 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/shard-1-001/]
   [junit4]   2> 439788 T4991 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/shard-1-001/lib
   [junit4]   2> 439788 T4991 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/shard-1-001/lib).
   [junit4]   2> 439801 T4991 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 439801 T4991 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 439802 T4991 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 439802 T4991 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 439802 T4991 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 439803 T4991 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 439803 T4991 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 439803 T4991 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 439804 T4991 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 439804 T4991 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 439804 T4991 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 439805 T4991 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 439805 T4991 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 439806 T4991 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 439806 T4991 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 439806 T4991 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:48272/solr
   [junit4]   2> 439807 T4991 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 439807 T4991 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 439808 T4991 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 439810 T5041 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1521d66d name:ZooKeeperConnection Watcher:127.0.0.1:48272 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 439810 T4991 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 439811 T4991 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 439813 T4991 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 439814 T5044 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@bf57f1 name:ZooKeeperConnection Watcher:127.0.0.1:48272/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 439815 T4991 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 439819 T4991 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 440823 T4991 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:48291_hg%2Fh
   [junit4]   2> 440824 T4991 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:48291_hg%2Fh
   [junit4]   2> 440826 T4991 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 440829 T4991 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/shard-1-001/cores
   [junit4]   1> CORE DESCRIPTOR: {loadOnStartup=true, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/shard-1-001/cores/collection1/, config=solrconfig.xml, schema=schema.xml, name=collection1, collection=collection1, transient=false, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/shard-1-001/cores/collection1, shard=, dataDir=data/, coreNodeName=}
   [junit4]   2> 440832 T4991 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/shard-1-001/cores/collection1/
   [junit4]   2> 440832 T4991 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 440834 T5045 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 440834 T5045 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 440835 T5045 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 440835 T5017 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 440836 T5018 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48291/hg/h",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:48291_hg%2Fh",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1"} current state version: 4
   [junit4]   2> 440837 T5018 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48291/hg/h",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:48291_hg%2Fh",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "core":"collection1"}
   [junit4]   2> 440837 T5018 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 440837 T5018 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 440838 T5017 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 440838 T5044 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 440838 T5029 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 441835 T5045 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 441836 T5045 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 441836 T5045 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 441837 T5045 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 441837 T5045 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 441838 T5045 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/shard-1-001/cores/collection1/'
   [junit4]   2> 441853 T5045 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 441855 T5045 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 441866 T5045 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 441888 T5045 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 441935 T5045 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 441936 T5045 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 441947 T5045 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 442215 T5045 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 442227 T5045 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 442228 T5045 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 442237 T5045 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 442240 T5045 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 442243 T5045 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 442244 T5045 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 442244 T5045 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 442244 T5045 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 442245 T5045 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 442245 T5045 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 442245 T5045 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 442245 T5045 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 442245 T5045 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 442246 T5045 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5208513
   [junit4]   2> 442246 T5045 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/shard-1-001/cores/collection1/data
   [junit4]   2> 442247 T5045 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 442247 T5045 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 442247 T5045 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 244FE66A6BD041BA-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 442247 T5045 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=16, maxMergeAtOnceExplicit=20, maxMergedSegmentMB=25.1201171875, floorSegmentMB=1.1103515625, forceMergeDeletesPctAllowed=27.54578614969428, segmentsPerTier=35.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 442248 T5045 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@20599bed lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5b0bf532),segFN=segments_1,generation=1}
   [junit4]   2> 442248 T5045 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 442254 T5045 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 442254 T5045 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 442254 T5045 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 442254 T5045 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 442254 T5045 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 442255 T5045 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 442255 T5045 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 442255 T5045 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 442255 T5045 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 442267 T5045 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 442269 T5045 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 442271 T5045 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 442272 T5045 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 442278 T5045 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/segments,/admin/mbeans,/admin/system,/admin/ping,/admin/luke,/admin/logging,/admin/plugins,/admin/properties,/admin/file,/update/json,/schema,/get,/config,/update/csv,/admin/threads,/update,/update/json/docs,/replication,standard
   [junit4]   2> 442280 T5045 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 442281 T5045 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 442281 T5045 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 442282 T5045 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 442282 T5045 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 442283 T5045 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=11, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7170157941624185]
   [junit4]   2> 442283 T5045 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@20599bed lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@5b0bf532),segFN=segments_1,generation=1}
   [junit4]   2> 442283 T5045 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 442284 T5045 oass.SolrIndexSearcher.<init> Opening Searcher@2323bf05[collection1] main
   [junit4]   2> 442284 T5045 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 442284 T5045 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 442285 T5045 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 442285 T5045 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 442285 T5045 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 442285 T5045 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 442286 T5045 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 442286 T5045 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 442286 T5045 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 442286 T5045 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 442287 T5046 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2323bf05[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 442288 T5045 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 442289 T5049 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:48291/hg/h collection:collection1 shard:shard2
   [junit4]   2> 442289 T4991 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 442289 T4991 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 442290 T5049 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard2/election
   [junit4]   2> 442294 T5049 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard2
   [junit4]   2> 442295 T5017 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 442295 T5049 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 442295 T5049 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C325 name=collection1 org.apache.solr.core.SolrCore@58216476 url=http://127.0.0.1:48291/hg/h/collection1 node=127.0.0.1:48291_hg%2Fh C325_STATE=coll:collection1 core:collection1 props:{base_url=http://127.0.0.1:48291/hg/h, state=down, node_name=127.0.0.1:48291_hg%2Fh, core=collection1}
   [junit4]   2> 442296 T5049 C325 P48291 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:48291/hg/h/collection1/
   [junit4]   2> 442296 T5018 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 442296 T5049 C325 P48291 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 442296 T5049 C325 P48291 oasc.SyncStrategy.syncToMe http://127.0.0.1:48291/hg/h/collection1/ has no replicas
   [junit4]   2> 442296 T5049 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:48291/hg/h/collection1/ shard2
   [junit4]   2> 442297 T5049 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard2
   [junit4]   2> 442300 T5017 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 442300 T5018 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:48291/hg/h",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 5
   [junit4]   2> 442403 T5029 oasc

[...truncated too long message...]

ncoding=ISO-8859-1
   [junit4] ERROR   62.0s J2 | RecoveryAfterSoftCommitTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22082/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([244FE66A6BD041BA:AC1BD9B0C52C2C42]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
   [junit4]    > 	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22082/collection1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	... 54 more
   [junit4]   2> 2164084 T6477 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 244FE66A6BD041BA-001
   [junit4]   2> 62019 T6476 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {range_facet_l=PostingsFormat(name=LuceneFixedGap), intDefault=PostingsFormat(name=LuceneFixedGap), timestamp=PostingsFormat(name=LuceneFixedGap), id=PostingsFormat(name=Direct), _version_=PostingsFormat(name=LuceneFixedGap), text=PostingsFormat(name=Memory doPackFST= false), a_t=PostingsFormat(name=LuceneFixedGap), multiDefault=PostingsFormat(name=Memory doPackFST= true)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=DefaultSimilarity, locale=es_AR, timezone=Pacific/Niue
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=113603472,total=412090368
   [junit4]   2> NOTE: All tests run in this JVM: [TestPHPSerializedResponseWriter, TestInitQParser, TestSolr4Spatial2, LukeRequestHandlerTest, TestBM25SimilarityFactory, TestManagedSchemaDynamicFieldResource, TestFoldingMultitermQuery, TestComponentsName, DocumentBuilderTest, TestSerializedLuceneMatchVersion, TestNoOpRegenerator, CacheHeaderTest, TestExactSharedStatsCache, TestSimpleQParserPlugin, HardAutoCommitTest, CloudExitableDirectoryReaderTest, SpatialFilterTest, CurrencyFieldOpenExchangeTest, LeaderFailoverAfterPartitionTest, TestReload, TestBlendedInfixSuggestions, DirectUpdateHandlerTest, TestSolrConfigHandler, TestRemoteStreaming, QueryEqualityTest, TestSuggestSpellingConverter, TestCloudInspectUtil, AnalyticsQueryTest, TestReqParamsAPI, TestExceedMaxTermLength, BadCopyFieldTest, TestRandomDVFaceting, SolrIndexSplitterTest, SpatialRPTFieldTypeTest, StandardRequestHandlerTest, TestReloadAndDeleteDocs, StatsComponentTest, TestLuceneMatchVersion, TestReplicaProperties, TestFastWriter, TestClassNameShortening, MoreLikeThisHandlerTest, SolrRequestParserTest, TestSolrQueryParser, SharedFSAutoReplicaFailoverUtilsTest, DirectUpdateHandlerOptimizeTest, SOLR749Test, TestQuerySenderNoQuery, LeaderElectionIntegrationTest, BadComponentTest, DistribCursorPagingTest, BufferStoreTest, TestBinaryField, ChaosMonkeyNothingIsSafeTest, TestOverriddenPrefixQueryForCustomFieldType, SuggestComponentTest, TestDistributedGrouping, TestFaceting, TestRecovery, DistributedSpellCheckComponentTest, ZkControllerTest, TestRealTimeGet, TestStressReorder, DistributedTermsComponentTest, SolrCmdDistributorTest, ConvertedLegacyTest, TestFiltering, TestIndexSearcher, SignatureUpdateProcessorFactoryTest, TestExtendedDismaxParser, TestStressLucene, PolyFieldTest, NoCacheHeaderTest, SchemaVersionSpecificBehaviorTest, FieldMutatingUpdateProcessorTest, DocValuesMultiTest, TestWriterPerf, TestReversedWildcardFilterFactory, PrimitiveFieldTypeTest, RequiredFieldsTest, LoggingHandlerTest, SolrPluginUtilsTest, TestJmxIntegration, UpdateRequestProcessorFactoryTest, MBeansHandlerTest, TestAnalyzedSuggestions, BinaryUpdateRequestHandlerTest, PingRequestHandlerTest, TestQuerySenderListener, TestSolrIndexConfig, SampleTest, NumericFieldsTest, TestFuzzyAnalyzedSuggestions, SynonymTokenizerTest, TestXIncludeConfig, TestDFRSimilarityFactory, ResourceLoaderTest, PluginInfoTest, TestFastLRUCache, TestSystemIdResolver, PrimUtilsTest, DateFieldTest, UUIDFieldTest, CircularListTest, DistributedMLTComponentTest, TestHighlightDedupGrouping, ActionThrottleTest, AssignTest, CollectionsAPIAsyncDistributedZkTest, ConnectionManagerTest, CustomCollectionTest, DeleteShardTest, DistribJoinFromCollectionTest, LeaderInitiatedRecoveryOnCommitTest, MultiThreadedOCPTest, RecoveryAfterSoftCommitTest]
   [junit4] Completed on J2 in 62.55s, 1 test, 1 error <<< FAILURES!

[...truncated 323 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 472 suites, 1882 tests, 3 errors, 1 failure, 58 ignored (9 assumptions)

Total time: 153 minutes 52 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 568885544 bytes
Compression is 0.0%
Took 1 min 42 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 786 - Still Failing

Posted by Robert Muir <rc...@gmail.com>.
The problem is with NIGHTLY we generate many more documents, if we get
unlucky and get something like MemoryPostings also are indexing stuff
like payloads, we get huge FSTs (the outputs have big postings lists)

I don't want to lose coverage for Nightly/Direct so I removed the
NIGHTLY conditional logic in TestDuelingCodecs and spun off a separate
@Nightly subclass that just excludes the memory-hungry ones.

On Tue, Mar 31, 2015 at 10:15 PM, Robert Muir <rc...@gmail.com> wrote:
> I can reproduce it with current 5.x, it hits OOME after about 15 minutes.
>
> On Sat, Mar 14, 2015 at 11:01 AM, Apache Jenkins Server
> <je...@builds.apache.org> wrote:
>> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/786/
>>
>> 1 tests failed.
>> REGRESSION:  org.apache.lucene.index.TestDuelingCodecs.testEquals
>>
>> Error Message:
>> Java heap space
>>
>> Stack Trace:
>> java.lang.OutOfMemoryError: Java heap space
>>         at __randomizedtesting.SeedInfo.seed([8353178BBB8D1BE0:873C441075B44BE2]:0)
>>         at org.apache.lucene.util.fst.BytesStore.skipBytes(BytesStore.java:307)
>>         at org.apache.lucene.util.fst.FST.addNode(FST.java:801)
>>         at org.apache.lucene.util.fst.NodeHash.add(NodeHash.java:126)
>>         at org.apache.lucene.util.fst.Builder.compileNode(Builder.java:189)
>>         at org.apache.lucene.util.fst.Builder.freezeTail(Builder.java:277)
>>         at org.apache.lucene.util.fst.Builder.add(Builder.java:381)
>>         at org.apache.lucene.codecs.memory.MemoryPostingsFormat$TermsWriter.finishTerm(MemoryPostingsFormat.java:257)
>>         at org.apache.lucene.codecs.memory.MemoryPostingsFormat$TermsWriter.access$500(MemoryPostingsFormat.java:112)
>>         at org.apache.lucene.codecs.memory.MemoryPostingsFormat$MemoryFieldsConsumer.write(MemoryPostingsFormat.java:399)
>>         at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:198)
>>         at org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
>>         at org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:186)
>>         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:95)
>>         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3933)
>>         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3514)
>>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
>>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:1803)
>>         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4572)
>>         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:704)
>>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4598)
>>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4589)
>>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1351)
>>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1138)
>>         at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:152)
>>         at org.apache.lucene.index.TestDuelingCodecs.createRandomIndex(TestDuelingCodecs.java:137)
>>         at org.apache.lucene.index.TestDuelingCodecs.testEquals(TestDuelingCodecs.java:149)
>>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>         at java.lang.reflect.Method.invoke(Method.java:606)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
>>
>>
>>
>>
>> Build Log:
>> [...truncated 1706 lines...]
>>    [junit4] Suite: org.apache.lucene.index.TestDuelingCodecs
>>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestDuelingCodecs -Dtests.method=testEquals -Dtests.seed=8353178BBB8D1BE0 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=fr_BE -Dtests.timezone=Africa/Timbuktu -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
>>    [junit4] ERROR   1667s J2 | TestDuelingCodecs.testEquals <<<
>>    [junit4]    > Throwable #1: java.lang.OutOfMemoryError: Java heap space
>>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([8353178BBB8D1BE0:873C441075B44BE2]:0)
>>    [junit4]    >        at org.apache.lucene.util.fst.BytesStore.skipBytes(BytesStore.java:307)
>>    [junit4]    >        at org.apache.lucene.util.fst.FST.addNode(FST.java:801)
>>    [junit4]    >        at org.apache.lucene.util.fst.NodeHash.add(NodeHash.java:126)
>>    [junit4]    >        at org.apache.lucene.util.fst.Builder.compileNode(Builder.java:189)
>>    [junit4]    >        at org.apache.lucene.util.fst.Builder.freezeTail(Builder.java:277)
>>    [junit4]    >        at org.apache.lucene.util.fst.Builder.add(Builder.java:381)
>>    [junit4]    >        at org.apache.lucene.codecs.memory.MemoryPostingsFormat$TermsWriter.finishTerm(MemoryPostingsFormat.java:257)
>>    [junit4]    >        at org.apache.lucene.codecs.memory.MemoryPostingsFormat$TermsWriter.access$500(MemoryPostingsFormat.java:112)
>>    [junit4]    >        at org.apache.lucene.codecs.memory.MemoryPostingsFormat$MemoryFieldsConsumer.write(MemoryPostingsFormat.java:399)
>>    [junit4]    >        at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:198)
>>    [junit4]    >        at org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
>>    [junit4]    >        at org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:186)
>>    [junit4]    >        at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:95)
>>    [junit4]    >        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3933)
>>    [junit4]    >        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3514)
>>    [junit4]    >        at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
>>    [junit4]    >        at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:1803)
>>    [junit4]    >        at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4572)
>>    [junit4]    >        at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:704)
>>    [junit4]    >        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4598)
>>    [junit4]    >        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4589)
>>    [junit4]    >        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1351)
>>    [junit4]    >        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1138)
>>    [junit4]    >        at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:152)
>>    [junit4]    >        at org.apache.lucene.index.TestDuelingCodecs.createRandomIndex(TestDuelingCodecs.java:137)
>>    [junit4]    >        at org.apache.lucene.index.TestDuelingCodecs.testEquals(TestDuelingCodecs.java:149)
>>    [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J2/temp/lucene.index.TestDuelingCodecs 8353178BBB8D1BE0-001
>>    [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {body=DFR I(ne)BZ(0.3), titleTokenized=DFR I(ne)LZ(0.3)}, locale=fr_BE, timezone=Africa/Timbuktu
>>    [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=151816600,total=475004928
>>    [junit4]   2> NOTE: All tests run in this JVM: [TestPrefixCodedTerms, TestCodecs, TestSearcherManager, TestFrequencyTrackingRingBuffer, TestMixedCodecs, TestBlockPostingsFormat2, TestSearchForDuplicates, TestDocIdSet, TestDoc, TestBinaryDocValuesUpdates, Test2BPositions, TestConjunctionDISI, TestOmitPositions, TestPostingsEnum, TestCrash, TestPrefixFilter, TestQueryRescorer, TestRateLimiter, TestMinimize, TestAtomicUpdate, TestSimilarity2, TestBytesRefArray, TestStringHelper, TestCustomNorms, TestSortRandom, TestSearch, TestIndexWriterExceptions2, TestCloseableThreadLocal, TestDeterminism, TestTermVectorsReader, Test2BPostings, TestStressNRT, TestCollectionUtil, Test2BSortedDocValues, TestCompressingStoredFieldsFormat, TestTragicIndexWriterDeadlock, TestTermsEnum2, TestPerFieldPostingsFormat, TestWorstCaseTestBehavior, TestRAMDirectory, TestDirectory, TestIndexWriterConfig, Nested1, Nested2, Nested3, TestTerms, TestDuelingCodecs]
>>    [junit4] Completed on J2 in 1685.39s, 2 tests, 1 error <<< FAILURES!
>>
>> [...truncated 22 lines...]
>>    [junit4] JVM J2: stdout was not empty, see: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/temp/junit4-J2-20150314_133848_781.sysout
>>    [junit4] >>> JVM J2: stdout (verbatim) ----
>>    [junit4] java.lang.OutOfMemoryError: Java heap space
>>    [junit4] Dumping heap to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/heapdumps/java_pid84537.hprof ...
>>    [junit4] Heap dump file created [422813476 bytes in 0.708 secs]
>>    [junit4] <<< JVM J2: EOF ----
>>
>> [...truncated 11 lines...]
>> BUILD FAILED
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:49: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 421 suites, 3375 tests, 1 error, 37 ignored (27 assumptions)
>>
>> Total time: 54 minutes 46 seconds
>> Build step 'Invoke Ant' marked build as failure
>> Archiving artifacts
>> Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
>> Archived 6 artifacts
>> Archive block size is 32768
>> Received 0 blocks and 426409874 bytes
>> Compression is 0.0%
>> Took 1 min 23 sec
>> Recording test results
>> Email was triggered for: Failure
>> Sending email for trigger: Failure
>>
>>
>>
>>
>> ---------------------------------------------------------------------
>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>> For additional commands, e-mail: dev-help@lucene.apache.org

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


Re: [JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 786 - Still Failing

Posted by Robert Muir <rc...@gmail.com>.
I can reproduce it with current 5.x, it hits OOME after about 15 minutes.

On Sat, Mar 14, 2015 at 11:01 AM, Apache Jenkins Server
<je...@builds.apache.org> wrote:
> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/786/
>
> 1 tests failed.
> REGRESSION:  org.apache.lucene.index.TestDuelingCodecs.testEquals
>
> Error Message:
> Java heap space
>
> Stack Trace:
> java.lang.OutOfMemoryError: Java heap space
>         at __randomizedtesting.SeedInfo.seed([8353178BBB8D1BE0:873C441075B44BE2]:0)
>         at org.apache.lucene.util.fst.BytesStore.skipBytes(BytesStore.java:307)
>         at org.apache.lucene.util.fst.FST.addNode(FST.java:801)
>         at org.apache.lucene.util.fst.NodeHash.add(NodeHash.java:126)
>         at org.apache.lucene.util.fst.Builder.compileNode(Builder.java:189)
>         at org.apache.lucene.util.fst.Builder.freezeTail(Builder.java:277)
>         at org.apache.lucene.util.fst.Builder.add(Builder.java:381)
>         at org.apache.lucene.codecs.memory.MemoryPostingsFormat$TermsWriter.finishTerm(MemoryPostingsFormat.java:257)
>         at org.apache.lucene.codecs.memory.MemoryPostingsFormat$TermsWriter.access$500(MemoryPostingsFormat.java:112)
>         at org.apache.lucene.codecs.memory.MemoryPostingsFormat$MemoryFieldsConsumer.write(MemoryPostingsFormat.java:399)
>         at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:198)
>         at org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
>         at org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:186)
>         at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:95)
>         at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3933)
>         at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3514)
>         at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
>         at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:1803)
>         at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4572)
>         at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:704)
>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4598)
>         at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4589)
>         at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1351)
>         at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1138)
>         at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:152)
>         at org.apache.lucene.index.TestDuelingCodecs.createRandomIndex(TestDuelingCodecs.java:137)
>         at org.apache.lucene.index.TestDuelingCodecs.testEquals(TestDuelingCodecs.java:149)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
>
>
>
>
> Build Log:
> [...truncated 1706 lines...]
>    [junit4] Suite: org.apache.lucene.index.TestDuelingCodecs
>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestDuelingCodecs -Dtests.method=testEquals -Dtests.seed=8353178BBB8D1BE0 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=fr_BE -Dtests.timezone=Africa/Timbuktu -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
>    [junit4] ERROR   1667s J2 | TestDuelingCodecs.testEquals <<<
>    [junit4]    > Throwable #1: java.lang.OutOfMemoryError: Java heap space
>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([8353178BBB8D1BE0:873C441075B44BE2]:0)
>    [junit4]    >        at org.apache.lucene.util.fst.BytesStore.skipBytes(BytesStore.java:307)
>    [junit4]    >        at org.apache.lucene.util.fst.FST.addNode(FST.java:801)
>    [junit4]    >        at org.apache.lucene.util.fst.NodeHash.add(NodeHash.java:126)
>    [junit4]    >        at org.apache.lucene.util.fst.Builder.compileNode(Builder.java:189)
>    [junit4]    >        at org.apache.lucene.util.fst.Builder.freezeTail(Builder.java:277)
>    [junit4]    >        at org.apache.lucene.util.fst.Builder.add(Builder.java:381)
>    [junit4]    >        at org.apache.lucene.codecs.memory.MemoryPostingsFormat$TermsWriter.finishTerm(MemoryPostingsFormat.java:257)
>    [junit4]    >        at org.apache.lucene.codecs.memory.MemoryPostingsFormat$TermsWriter.access$500(MemoryPostingsFormat.java:112)
>    [junit4]    >        at org.apache.lucene.codecs.memory.MemoryPostingsFormat$MemoryFieldsConsumer.write(MemoryPostingsFormat.java:399)
>    [junit4]    >        at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:198)
>    [junit4]    >        at org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
>    [junit4]    >        at org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:186)
>    [junit4]    >        at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:95)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3933)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3514)
>    [junit4]    >        at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:1803)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4572)
>    [junit4]    >        at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:704)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4598)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4589)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1351)
>    [junit4]    >        at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1138)
>    [junit4]    >        at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:152)
>    [junit4]    >        at org.apache.lucene.index.TestDuelingCodecs.createRandomIndex(TestDuelingCodecs.java:137)
>    [junit4]    >        at org.apache.lucene.index.TestDuelingCodecs.testEquals(TestDuelingCodecs.java:149)
>    [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J2/temp/lucene.index.TestDuelingCodecs 8353178BBB8D1BE0-001
>    [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {body=DFR I(ne)BZ(0.3), titleTokenized=DFR I(ne)LZ(0.3)}, locale=fr_BE, timezone=Africa/Timbuktu
>    [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=151816600,total=475004928
>    [junit4]   2> NOTE: All tests run in this JVM: [TestPrefixCodedTerms, TestCodecs, TestSearcherManager, TestFrequencyTrackingRingBuffer, TestMixedCodecs, TestBlockPostingsFormat2, TestSearchForDuplicates, TestDocIdSet, TestDoc, TestBinaryDocValuesUpdates, Test2BPositions, TestConjunctionDISI, TestOmitPositions, TestPostingsEnum, TestCrash, TestPrefixFilter, TestQueryRescorer, TestRateLimiter, TestMinimize, TestAtomicUpdate, TestSimilarity2, TestBytesRefArray, TestStringHelper, TestCustomNorms, TestSortRandom, TestSearch, TestIndexWriterExceptions2, TestCloseableThreadLocal, TestDeterminism, TestTermVectorsReader, Test2BPostings, TestStressNRT, TestCollectionUtil, Test2BSortedDocValues, TestCompressingStoredFieldsFormat, TestTragicIndexWriterDeadlock, TestTermsEnum2, TestPerFieldPostingsFormat, TestWorstCaseTestBehavior, TestRAMDirectory, TestDirectory, TestIndexWriterConfig, Nested1, Nested2, Nested3, TestTerms, TestDuelingCodecs]
>    [junit4] Completed on J2 in 1685.39s, 2 tests, 1 error <<< FAILURES!
>
> [...truncated 22 lines...]
>    [junit4] JVM J2: stdout was not empty, see: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/temp/junit4-J2-20150314_133848_781.sysout
>    [junit4] >>> JVM J2: stdout (verbatim) ----
>    [junit4] java.lang.OutOfMemoryError: Java heap space
>    [junit4] Dumping heap to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/heapdumps/java_pid84537.hprof ...
>    [junit4] Heap dump file created [422813476 bytes in 0.708 secs]
>    [junit4] <<< JVM J2: EOF ----
>
> [...truncated 11 lines...]
> BUILD FAILED
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:49: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 421 suites, 3375 tests, 1 error, 37 ignored (27 assumptions)
>
> Total time: 54 minutes 46 seconds
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
> Archived 6 artifacts
> Archive block size is 32768
> Received 0 blocks and 426409874 bytes
> Compression is 0.0%
> Took 1 min 23 sec
> Recording test results
> Email was triggered for: Failure
> Sending email for trigger: Failure
>
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 786 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/786/

1 tests failed.
REGRESSION:  org.apache.lucene.index.TestDuelingCodecs.testEquals

Error Message:
Java heap space

Stack Trace:
java.lang.OutOfMemoryError: Java heap space
	at __randomizedtesting.SeedInfo.seed([8353178BBB8D1BE0:873C441075B44BE2]:0)
	at org.apache.lucene.util.fst.BytesStore.skipBytes(BytesStore.java:307)
	at org.apache.lucene.util.fst.FST.addNode(FST.java:801)
	at org.apache.lucene.util.fst.NodeHash.add(NodeHash.java:126)
	at org.apache.lucene.util.fst.Builder.compileNode(Builder.java:189)
	at org.apache.lucene.util.fst.Builder.freezeTail(Builder.java:277)
	at org.apache.lucene.util.fst.Builder.add(Builder.java:381)
	at org.apache.lucene.codecs.memory.MemoryPostingsFormat$TermsWriter.finishTerm(MemoryPostingsFormat.java:257)
	at org.apache.lucene.codecs.memory.MemoryPostingsFormat$TermsWriter.access$500(MemoryPostingsFormat.java:112)
	at org.apache.lucene.codecs.memory.MemoryPostingsFormat$MemoryFieldsConsumer.write(MemoryPostingsFormat.java:399)
	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:198)
	at org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
	at org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:186)
	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:95)
	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3933)
	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3514)
	at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
	at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:1803)
	at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4572)
	at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:704)
	at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4598)
	at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4589)
	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1351)
	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1138)
	at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:152)
	at org.apache.lucene.index.TestDuelingCodecs.createRandomIndex(TestDuelingCodecs.java:137)
	at org.apache.lucene.index.TestDuelingCodecs.testEquals(TestDuelingCodecs.java:149)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)




Build Log:
[...truncated 1706 lines...]
   [junit4] Suite: org.apache.lucene.index.TestDuelingCodecs
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestDuelingCodecs -Dtests.method=testEquals -Dtests.seed=8353178BBB8D1BE0 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=fr_BE -Dtests.timezone=Africa/Timbuktu -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   1667s J2 | TestDuelingCodecs.testEquals <<<
   [junit4]    > Throwable #1: java.lang.OutOfMemoryError: Java heap space
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([8353178BBB8D1BE0:873C441075B44BE2]:0)
   [junit4]    > 	at org.apache.lucene.util.fst.BytesStore.skipBytes(BytesStore.java:307)
   [junit4]    > 	at org.apache.lucene.util.fst.FST.addNode(FST.java:801)
   [junit4]    > 	at org.apache.lucene.util.fst.NodeHash.add(NodeHash.java:126)
   [junit4]    > 	at org.apache.lucene.util.fst.Builder.compileNode(Builder.java:189)
   [junit4]    > 	at org.apache.lucene.util.fst.Builder.freezeTail(Builder.java:277)
   [junit4]    > 	at org.apache.lucene.util.fst.Builder.add(Builder.java:381)
   [junit4]    > 	at org.apache.lucene.codecs.memory.MemoryPostingsFormat$TermsWriter.finishTerm(MemoryPostingsFormat.java:257)
   [junit4]    > 	at org.apache.lucene.codecs.memory.MemoryPostingsFormat$TermsWriter.access$500(MemoryPostingsFormat.java:112)
   [junit4]    > 	at org.apache.lucene.codecs.memory.MemoryPostingsFormat$MemoryFieldsConsumer.write(MemoryPostingsFormat.java:399)
   [junit4]    > 	at org.apache.lucene.codecs.perfield.PerFieldPostingsFormat$FieldsWriter.write(PerFieldPostingsFormat.java:198)
   [junit4]    > 	at org.apache.lucene.codecs.FieldsConsumer.merge(FieldsConsumer.java:105)
   [junit4]    > 	at org.apache.lucene.index.SegmentMerger.mergeTerms(SegmentMerger.java:186)
   [junit4]    > 	at org.apache.lucene.index.SegmentMerger.merge(SegmentMerger.java:95)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.mergeMiddle(IndexWriter.java:3933)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.merge(IndexWriter.java:3514)
   [junit4]    > 	at org.apache.lucene.index.SerialMergeScheduler.merge(SerialMergeScheduler.java:40)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.maybeMerge(IndexWriter.java:1803)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.doAfterSegmentFlushed(IndexWriter.java:4572)
   [junit4]    > 	at org.apache.lucene.index.DocumentsWriter$MergePendingEvent.process(DocumentsWriter.java:704)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4598)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:4589)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.updateDocument(IndexWriter.java:1351)
   [junit4]    > 	at org.apache.lucene.index.IndexWriter.addDocument(IndexWriter.java:1138)
   [junit4]    > 	at org.apache.lucene.index.RandomIndexWriter.addDocument(RandomIndexWriter.java:152)
   [junit4]    > 	at org.apache.lucene.index.TestDuelingCodecs.createRandomIndex(TestDuelingCodecs.java:137)
   [junit4]    > 	at org.apache.lucene.index.TestDuelingCodecs.testEquals(TestDuelingCodecs.java:149)
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J2/temp/lucene.index.TestDuelingCodecs 8353178BBB8D1BE0-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {body=DFR I(ne)BZ(0.3), titleTokenized=DFR I(ne)LZ(0.3)}, locale=fr_BE, timezone=Africa/Timbuktu
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=151816600,total=475004928
   [junit4]   2> NOTE: All tests run in this JVM: [TestPrefixCodedTerms, TestCodecs, TestSearcherManager, TestFrequencyTrackingRingBuffer, TestMixedCodecs, TestBlockPostingsFormat2, TestSearchForDuplicates, TestDocIdSet, TestDoc, TestBinaryDocValuesUpdates, Test2BPositions, TestConjunctionDISI, TestOmitPositions, TestPostingsEnum, TestCrash, TestPrefixFilter, TestQueryRescorer, TestRateLimiter, TestMinimize, TestAtomicUpdate, TestSimilarity2, TestBytesRefArray, TestStringHelper, TestCustomNorms, TestSortRandom, TestSearch, TestIndexWriterExceptions2, TestCloseableThreadLocal, TestDeterminism, TestTermVectorsReader, Test2BPostings, TestStressNRT, TestCollectionUtil, Test2BSortedDocValues, TestCompressingStoredFieldsFormat, TestTragicIndexWriterDeadlock, TestTermsEnum2, TestPerFieldPostingsFormat, TestWorstCaseTestBehavior, TestRAMDirectory, TestDirectory, TestIndexWriterConfig, Nested1, Nested2, Nested3, TestTerms, TestDuelingCodecs]
   [junit4] Completed on J2 in 1685.39s, 2 tests, 1 error <<< FAILURES!

[...truncated 22 lines...]
   [junit4] JVM J2: stdout was not empty, see: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/temp/junit4-J2-20150314_133848_781.sysout
   [junit4] >>> JVM J2: stdout (verbatim) ----
   [junit4] java.lang.OutOfMemoryError: Java heap space
   [junit4] Dumping heap to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/heapdumps/java_pid84537.hprof ...
   [junit4] Heap dump file created [422813476 bytes in 0.708 secs]
   [junit4] <<< JVM J2: EOF ----

[...truncated 11 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:49: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 421 suites, 3375 tests, 1 error, 37 ignored (27 assumptions)

Total time: 54 minutes 46 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 6 artifacts
Archive block size is 32768
Received 0 blocks and 426409874 bytes
Compression is 0.0%
Took 1 min 23 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 785 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/785/

3 tests failed.
FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:23458/s/hy/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:23458/s/hy/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([BBA65530E0D7CF12:33F26AEA4E2BA2EA]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([BBA65530E0D7CF12:33F26AEA4E2BA2EA]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([BBA65530E0D7CF12:33F26AEA4E2BA2EA]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 10254 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/init-core-data-001
   [junit4]   2> 328931 T1662 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /bl_/xw
   [junit4]   2> 328939 T1662 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 328940 T1663 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 329040 T1662 oasc.ZkTestServer.run start zk server on port:16266
   [junit4]   2> 329041 T1662 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 329042 T1662 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 329054 T1670 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@289fd77 name:ZooKeeperConnection Watcher:127.0.0.1:16266 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 329055 T1662 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 329055 T1662 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 329056 T1662 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 329064 T1662 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 329066 T1662 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 329068 T1673 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7ebb8c6c name:ZooKeeperConnection Watcher:127.0.0.1:16266/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 329068 T1662 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 329069 T1662 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 329069 T1662 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 329077 T1662 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 329082 T1662 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 329086 T1662 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 329093 T1662 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 329094 T1662 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 329109 T1662 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 329110 T1662 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 329117 T1662 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 329118 T1662 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 329125 T1662 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 329126 T1662 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 329131 T1662 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 329133 T1662 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 329143 T1662 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 329144 T1662 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 329149 T1662 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 329150 T1662 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 329154 T1662 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 329155 T1662 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 329159 T1662 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 329161 T1662 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 329166 T1662 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 329168 T1662 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 329175 T1662 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 329177 T1662 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 332290 T1662 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/control-001/cores/collection1
   [junit4]   2> 332299 T1662 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 332311 T1662 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:16287
   [junit4]   2> 332313 T1662 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostPort=16286, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/tempDir-001/control/data, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/control-001/cores, hostContext=/bl_/xw}
   [junit4]   2> 332315 T1662 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 332316 T1662 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/control-001/'
   [junit4]   2> 332405 T1662 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/control-001/solr.xml
   [junit4]   2> 332488 T1662 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/control-001/cores
   [junit4]   2> 332488 T1662 oasc.CoreContainer.<init> New CoreContainer 785943869
   [junit4]   2> 332488 T1662 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/control-001/]
   [junit4]   2> 332489 T1662 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/control-001/lib
   [junit4]   2> 332490 T1662 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/control-001/lib).
   [junit4]   2> 332502 T1662 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 332502 T1662 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 332502 T1662 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 332503 T1662 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 332503 T1662 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 332503 T1662 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 332503 T1662 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 332504 T1662 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 332504 T1662 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 332504 T1662 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 332504 T1662 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 332505 T1662 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 332505 T1662 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 332506 T1662 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 332506 T1662 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 332506 T1662 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:16266/solr
   [junit4]   2> 332507 T1662 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 332507 T1662 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 332508 T1662 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 332510 T1685 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@87e258d name:ZooKeeperConnection Watcher:127.0.0.1:16266 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 332510 T1662 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 332511 T1662 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 332512 T1662 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 332514 T1688 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3f2c4159 name:ZooKeeperConnection Watcher:127.0.0.1:16266/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 332514 T1662 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 332515 T1662 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 332518 T1662 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 332520 T1662 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 332522 T1662 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 332524 T1662 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 332526 T1662 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 332527 T1662 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 332528 T1662 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 332530 T1662 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:16286_bl_%2Fxw
   [junit4]   2> 332530 T1662 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:16286_bl_%2Fxw
   [junit4]   2> 332532 T1662 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 332533 T1662 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 332535 T1662 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 332536 T1662 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:16286_bl_%2Fxw
   [junit4]   2> 332536 T1662 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 332538 T1662 oasc.Overseer.start Overseer (id=93471408720838659-127.0.0.1:16286_bl_%2Fxw-n_0000000000) starting
   [junit4]   2> 332540 T1662 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 332546 T1662 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 332546 T1690 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 332546 T1662 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 332547 T1689 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 332551 T1662 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/control-001/cores
   [junit4]   1> CORE DESCRIPTOR: {name=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/control-001/cores/collection1, shard=, transient=false, loadOnStartup=true, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/control-001/cores/collection1/, collection=control_collection, dataDir=data/, schema=schema.xml, config=solrconfig.xml}
   [junit4]   2> 332553 T1662 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/control-001/cores/collection1/
   [junit4]   2> 332554 T1662 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 332555 T1692 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 332555 T1692 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 332556 T1688 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 332556 T1692 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 332557 T1689 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:16286_bl_%2Fxw",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:16286/bl_/xw",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 0
   [junit4]   2> 332557 T1689 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:16286_bl_%2Fxw",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:16286/bl_/xw",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 332557 T1689 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 332558 T1689 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 332559 T1688 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 333557 T1692 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 333558 T1692 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 333560 T1692 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 333560 T1692 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 333562 T1692 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 333563 T1692 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/control-001/cores/collection1/'
   [junit4]   2> 333595 T1692 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 333599 T1692 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 333621 T1692 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 333731 T1692 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 333790 T1692 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 333792 T1692 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 333815 T1692 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 334663 T1692 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 334737 T1692 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 334757 T1692 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 334851 T1692 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 334887 T1692 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 334919 T1692 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 334923 T1692 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 334926 T1692 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 334926 T1692 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 334929 T1692 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 334929 T1692 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 334929 T1692 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 334930 T1692 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 334930 T1692 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/control-001/cores/collection1/data/
   [junit4]   2> 334931 T1692 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@74ac357e
   [junit4]   2> 334932 T1692 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/control-001/cores/collection1/data
   [junit4]   2> 334932 T1692 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/control-001/cores/collection1/data/index/
   [junit4]   2> 334933 T1692 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 334933 T1692 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/control-001/cores/collection1/data/index
   [junit4]   2> 334934 T1692 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=36, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=70.37890625, floorSegmentMB=1.7412109375, forceMergeDeletesPctAllowed=27.656709584766915, segmentsPerTier=22.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 334935 T1692 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@4f209da8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@664ea92d; maxCacheMB=0.4208984375 maxMergeSizeMB=0.5029296875)),segFN=segments_1,generation=1}
   [junit4]   2> 334935 T1692 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 334944 T1692 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 334945 T1692 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 334945 T1692 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 334945 T1692 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 334946 T1692 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 334946 T1692 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 334947 T1692 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 334947 T1692 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 334947 T1692 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 335033 T1692 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 335075 T1692 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 335088 T1692 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 335100 T1692 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 335109 T1692 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/threads,/update/csv,/get,/admin/segments,/admin/logging,/replication,/admin/mbeans,/schema,/admin/plugins,/admin/file,/admin/properties,/admin/system,/update/json,/admin/luke,/admin/ping,/update,/update/json/docs,standard,/config
   [junit4]   2> 335143 T1692 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 335233 T1692 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 335233 T1692 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 335260 T1692 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 335260 T1692 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 335262 T1692 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 335263 T1692 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@4f209da8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@664ea92d; maxCacheMB=0.4208984375 maxMergeSizeMB=0.5029296875)),segFN=segments_1,generation=1}
   [junit4]   2> 335263 T1692 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 335264 T1692 oass.SolrIndexSearcher.<init> Opening Searcher@7c22a27d[collection1] main
   [junit4]   2> 335264 T1692 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 335290 T1692 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 335292 T1692 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 335296 T1692 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 335297 T1692 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 335297 T1692 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 335301 T1692 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 335301 T1692 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 335301 T1692 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 335302 T1692 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 335304 T1693 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7c22a27d[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 335306 T1692 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 335308 T1696 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:16286/bl_/xw collection:control_collection shard:shard1
   [junit4]   2> 335310 T1662 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 335310 T1662 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 335313 T1696 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 335323 T1662 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 335324 T1662 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 335328 T1700 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@25712c3d name:ZooKeeperConnection Watcher:127.0.0.1:16266/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 335328 T1662 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 335329 T1662 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 335330 T1662 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 335335 T1696 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 335335 T1662 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 335338 T1688 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 335339 T1696 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 335339 T1696 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C150 name=collection1 org.apache.solr.core.SolrCore@7067f6a9 url=http://127.0.0.1:16286/bl_/xw/collection1 node=127.0.0.1:16286_bl_%2Fxw C150_STATE=coll:control_collection core:collection1 props:{state=down, core=collection1, node_name=127.0.0.1:16286_bl_%2Fxw, base_url=http://127.0.0.1:16286/bl_/xw}
   [junit4]   2> 335340 T1696 C150 P16286 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:16286/bl_/xw/collection1/
   [junit4]   2> 335340 T1696 C150 P16286 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 335340 T1696 C150 P16286 oasc.SyncStrategy.syncToMe http://127.0.0.1:16286/bl_/xw/collection1/ has no replicas
   [junit4]   2> 335341 T1696 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:16286/bl_/xw/collection1/ shard1
   [junit4]   2> 335341 T1689 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 335341 T1696 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 335345 T1700 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 335345 T1688 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 335353 T1688 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 335355 T1689 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:16286/bl_/xw",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 335460 T1700 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 335460 T1688 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 335508 T1696 oasc.ZkController.register We are http://127.0.0.1:16286/bl_/xw/collection1/ and leader is http://127.0.0.1:16286/bl_/xw/collection1/
   [junit4]   2> 335508 T1696 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:16286/bl_/xw
   [junit4]   2> 335508 T1696 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 335508 T1696 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 335509 T1696 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 335525 T1688 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 335526 T1689 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:16286_bl_%2Fxw",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:16286/bl_/xw",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 3
   [junit4]   2> 335529 T1689 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:16286_bl_%2Fxw",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:16286/bl_/xw",
   [junit4]   2> 	  "collection":"control_collection"}
   [junit4]   2> 335653 T1700 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 335654 T1688 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 342569 T1662 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-1-001/cores/collection1
   [junit4]   2> 342623 T1662 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-1-001
   [junit4]   2> 342624 T1662 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 342634 T1662 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:16338
   [junit4]   2> 342637 T1662 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, hostContext=/bl_/xw, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/tempDir-001/jetty1, hostPort=16337, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-1-001/cores}
   [junit4]   2> 342638 T1662 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 342638 T1662 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-1-001/'
   [junit4]   2> 342965 T1662 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-1-001/solr.xml
   [junit4]   2> 343128 T1662 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-1-001/cores
   [junit4]   2> 343129 T1662 oasc.CoreContainer.<init> New CoreContainer 785451259
   [junit4]   2> 343129 T1662 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-1-001/]
   [junit4]   2> 343129 T1662 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-1-001/lib
   [junit4]   2> 343132 T1662 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-1-001/lib).
   [junit4]   2> 343168 T1662 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 343169 T1662 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 343170 T1662 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 343170 T1662 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 343171 T1662 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 343171 T1662 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 343172 T1662 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 343172 T1662 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 343173 T1662 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 343173 T1662 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 343174 T1662 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 343175 T1662 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 343176 T1662 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 343176 T1662 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 343176 T1662 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 343177 T1662 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:16266/solr
   [junit4]   2> 343177 T1662 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 343178 T1662 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 343179 T1662 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 343181 T1712 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2303b2ad name:ZooKeeperConnection Watcher:127.0.0.1:16266 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 343182 T1662 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 343183 T1662 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 343186 T1662 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 343187 T1715 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@294fa094 name:ZooKeeperConnection Watcher:127.0.0.1:16266/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 343188 T1662 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 343192 T1662 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 344197 T1662 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:16337_bl_%2Fxw
   [junit4]   2> 344198 T1662 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:16337_bl_%2Fxw
   [junit4]   2> 344202 T1662 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 344206 T1662 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-1-001/cores
   [junit4]   1> CORE DESCRIPTOR: {transient=false, schema=schema.xml, dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-1-001/cores/collection1/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-1-001/cores/collection1, collection=collection1, loadOnStartup=true, shard=, coreNodeName=, name=collection1, config=solrconfig.xml}
   [junit4]   2> 344214 T1662 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-1-001/cores/collection1/
   [junit4]   2> 344214 T1662 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 344217 T1716 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 344217 T1716 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 344218 T1716 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 344218 T1688 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 344220 T1689 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:16337_bl_%2Fxw",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:16337/bl_/xw",
   [junit4]   2> 	  "collection":"collection1"} current state version: 4
   [junit4]   2> 344220 T1689 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:16337_bl_%2Fxw",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:16337/bl_/xw",
   [junit4]   2> 	  "collection":"collection1"}
   [junit4]   2> 344221 T1689 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 344221 T1689 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 344224 T1688 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 344224 T1700 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 344227 T1715 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 345219 T1716 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 345220 T1716 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 345221 T1716 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 345221 T1716 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 345222 T1716 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 345222 T1716 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-1-001/cores/collection1/'
   [junit4]   2> 345251 T1716 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 345257 T1716 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 345291 T1716 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 345349 T1716 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 345445 T1716 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 345447 T1716 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 345475 T1716 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 346185 T1716 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 346228 T1716 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 346236 T1716 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 346260 T1716 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 346268 T1716 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 346275 T1716 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 346276 T1716 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 346276 T1716 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 346276 T1716 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 346277 T1716 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 346277 T1716 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 346278 T1716 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 346278 T1716 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 346278 T1716 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 346278 T1716 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@74ac357e
   [junit4]   2> 346279 T1716 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-1-001/cores/collection1/data
   [junit4]   2> 346280 T1716 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 346280 T1716 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 346280 T1716 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-1-001/cores/collection1/data/index
   [junit4]   2> 346281 T1716 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=36, maxMergeAtOnceExplicit=19, maxMergedSegmentMB=70.37890625, floorSegmentMB=1.7412109375, forceMergeDeletesPctAllowed=27.656709584766915, segmentsPerTier=22.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 346281 T1716 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@677421e9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@795e7d0e; maxCacheMB=0.4208984375 maxMergeSizeMB=0.5029296875)),segFN=segments_1,generation=1}
   [junit4]   2> 346282 T1716 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 346288 T1716 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 346289 T1716 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 346289 T1716 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 346289 T1716 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 346290 T1716 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 346290 T1716 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 346290 T1716 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 346290 T1716 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 346291 T1716 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 346318 T1716 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 346321 T1716 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 346324 T1716 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 346326 T1716 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 346344 T1716 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /admin/threads,/update/csv,/get,/admin/segments,/admin/logging,/replication,/admin/mbeans,/schema,/admin/plugins,/admin/file,/admin/properties,/admin/system,/update/json,/admin/luke,/admin/ping,/update,/update/json/docs,standard,/config
   [junit4]   2> 346346 T1716 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 346348 T1716 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 346348 T1716 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 346349 T1716 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 346349 T1716 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 346350 T1716 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=34, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 346351 T1716 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@677421e9 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@795e7d0e; maxCacheMB=0.4208984375 maxMergeSizeMB=0.5029296875)),segFN=segments_1,generation=1}
   [junit4]   2> 346351 T1716 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 346352 T1716 oass.SolrIndexSearcher.<init> Opening Searcher@76ff1494[collection1] main
   [junit4]   2> 346352 T1716 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 346353 T1716 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 346353 T1716 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 346353 T1716 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 346353 T1716 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 346354 T1716 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 346354 T1716 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 346354 T1716 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 346354 T1716 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 346355 T1716 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 346356 T1717 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@76ff1494[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 346356 T1716 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 346357 T1720 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:16337/bl_/xw collection:collection1 shard:shard1
   [junit4]   2> 346357 T1662 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 346358 T1662 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 346358 T1720 oascc.SolrZkClient.makePath makePath: /collections/collection1/leader_elect/shard1/election
   [junit4]   2> 346362 T1720 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 346363 T1688 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 346363 T1720 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 346364 T1720 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C151 name=collection1 org.apache.solr.core.SolrCore@60d7f49a url=http://127.0.0.1:16337/bl_/xw/collection1 node=127.0.0.1:16337_bl_%2Fxw C151_STATE=coll:collection1 core:collection1 props:{state=down, core=collection1, node_name=127.0.0.1:16337_bl_%2Fxw, base_url=http://127.0.0.1:16337/bl_/xw}
   [junit4]   2> 346364 T1720 C151 P16337 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:16337/bl_/xw/collection1/
   [junit4]   2> 346364 T1689 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1"} current state version: 5
   [junit4]   2> 346364 T1720 C151 P16337 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 346364 T1720 C151 P16337 oasc.SyncStrategy.syncToMe http://127.0.0.1:16337/bl_/xw/collection1/ has no replicas
   [junit4]   2> 346365 T1720 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:16337/bl_/xw/collection1/ shard1
   [junit4]   2> 346365 T1720 oascc.SolrZkClient.makePath makePath: /collections/collection1/leaders/shard1
   [junit4]   2> 346365 T1715 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 346365 T1700 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 346365 T1688 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 346369 T1688 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 346369 T1689 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:16337/bl_/xw",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 6
   [junit4]   2> 346472 T1700 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 346472 T1688 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 346472 T1715 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 346522 T1720 oasc.ZkController.register We are http://127.0.0.1:16337/bl_/xw/collection1/ and leader is http://127.0.0.1:16337/bl_/xw/collection1/
   [junit4]   2> 346522 T1720 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:16337/bl_/xw
   [junit4]   2> 346523 T1720 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 346523 T1720 oasc.ZkController.publish publishing core=collection1 state=active collection=collection1
   [junit4]   2> 346523 T1720 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 346525 T1688 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 346526 T1689 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:16337_bl_%2Fxw",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:16337/bl_/xw",
   [junit4]   2> 	  "collection":"collection1"} current state version: 7
   [junit4]   2> 346528 T1689 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:16337_bl_%2Fxw",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:16337/bl_/xw",
   [junit4]   2> 	  "collection":"collection1"}
   [junit4]   2> 346636 T1688 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 346636 T1715 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 346636 T1700 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 352924 T1662 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-2-001/cores/collection1
   [junit4]   2> 352929 T1662 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 2 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-2-001
   [junit4]   2> 352930 T1662 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 352953 T1662 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:16382
   [junit4]   2> 352954 T1662 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/tempDir-001/jetty2, hostContext=/bl_/xw, hostPort=16381, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-2-001/cores}
   [junit4]   2> 352955 T1662 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 352955 T1662 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-2-001/'
   [junit4]   2> 353033 T1662 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-2-001/solr.xml
   [junit4]   2> 353100 T1662 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-2-001/cores
   [junit4]   2> 353100 T1662 oasc.CoreContainer.<init> New CoreContainer 1390748695
   [junit4]   2> 353101 T1662 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-2-001/]
   [junit4]   2> 353101 T1662 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-2-001/lib
   [junit4]   2> 353102 T1662 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest BBA65530E0D7CF12-001/shard-2-001/lib).
   [junit4]   2> 353113 T1662 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 353113 T1662 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 353113 T1662 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 353113 T1662 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 353114 T1662 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 353114 T1662 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 353114 T1662 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 353114 T1662 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 353115 T1662 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 353115 T1662 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 353115 T1662 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 353115 T1662 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 353116 T1662 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 353116 T1662 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 353116 T1662 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 353116 T1662 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:16266/solr
   [junit4]   2> 353117 T1662 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 353117 T1662 oascc

[...truncated too long message...]

junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> 2190087 T40950 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:23526/i_b/kc
   [junit4]   2> 2190087 T40950 oasc.SocketProxy.close WARN Closing 10 connections to: http://127.0.0.1:23539/i_b/kc
   [junit4]   2> 2190088 T40950 oasc.SocketProxy.close WARN Closing 3 connections to: http://127.0.0.1:23511/i_b/kc
   [junit4]   2> 2190088 T40950 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:23494/i_b/kc
   [junit4]   2> 2190088 T40950 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 2190089 T40950 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:23526/i_b/kc
   [junit4]   2> 2190089 T40950 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:23539/i_b/kc
   [junit4]   2> 2190089 T40950 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:23511/i_b/kc
   [junit4]   2> 2190089 T40950 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:23494/i_b/kc
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=BBA65530E0D7CF12 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=de_CH -Dtests.timezone=Africa/Lagos -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   69.8s J0 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([BBA65530E0D7CF12:33F26AEA4E2BA2EA]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:929)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:308)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:234)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:113)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:596)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:593)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 2190118 T40950 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest BBA65530E0D7CF12-001
   [junit4]   2> 69774 T40949 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=DummyCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=DUMMY, chunkSize=30969, maxDocsPerChunk=3, blockSize=778), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=DUMMY, chunkSize=30969, blockSize=778)), sim=DefaultSimilarity, locale=de_CH, timezone=Africa/Lagos
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=261273792,total=400556032
   [junit4]   2> NOTE: All tests run in this JVM: [TestFastLRUCache, DirectSolrSpellCheckerTest, TestRawResponseWriter, SpatialHeatmapFacetsTest, TestSort, RegexBoostProcessorTest, TestCoreContainer, TestUpdate, InfoHandlerTest, TestHdfsUpdateLog, SolrCmdDistributorTest, FastVectorHighlighterTest, RequestHandlersTest, CloudMLTQParserTest, ZkNodePropsTest, TestCloudManagedSchemaConcurrent, TestBlobHandler, ExternalFileFieldSortTest, TestRestManager, ShowFileRequestHandlerTest, TestTrackingShardHandlerFactory, TestDFRSimilarityFactory, OverseerTest, TestComplexPhraseQParserPlugin, CustomCollectionTest, HdfsSyncSliceTest, TestMissingGroups, HdfsCollectionsAPIDistributedZkTest, TestIBSimilarityFactory, TestCollationFieldDocValues, TestConfigSets, MinimalSchemaTest, PolyFieldTest, ShardRoutingCustomTest, TestSolrDeletionPolicy2, SSLMigrationTest, TestCollationField, DistributedQueueTest, ParsingFieldUpdateProcessorsTest, DOMUtilTest, DebugComponentTest, TestSolrJ, TestImplicitCoreProperties, TestUniqueKeyFieldResource, TestFaceting, TestReplicationHandlerBackup, TestManagedStopFilterFactory, SearchHandlerTest, TestPartialUpdateDeduplication, TestFieldCollectionResource, SimpleCollectionCreateDeleteTest, TestRandomFaceting, SegmentsInfoRequestHandlerTest, TimeZoneUtilsTest, SuggesterTest, TestCharFilters, RecoveryZkTest, LeaderElectionIntegrationTest, ShardRoutingTest, BasicZkTest, ClusterStateUpdateTest, ZkSolrClientTest, TestZkChroot, TestRandomDVFaceting, TestDistributedSearch, DeleteInactiveReplicaTest, ExternalCollectionsTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, ReplicationFactorTest]
   [junit4] Completed on J0 in 70.57s, 1 test, 1 error <<< FAILURES!

[...truncated 336 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 472 suites, 1883 tests, 3 errors, 58 ignored (9 assumptions)

Total time: 157 minutes 29 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 689586426 bytes
Compression is 0.0%
Took 2 min 10 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 784 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/784/

8 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test

Error Message:
shard3 is not consistent.  Got 110 from http://127.0.0.1:59031/collection1lastClient and got 47 from http://127.0.0.1:59057/collection1

Stack Trace:
java.lang.AssertionError: shard3 is not consistent.  Got 110 from http://127.0.0.1:59031/collection1lastClient and got 47 from http://127.0.0.1:59057/collection1
	at __randomizedtesting.SeedInfo.seed([6E6801B0270E8FC3:E63C3E6A89F2E23B]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1286)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1265)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:161)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1426174267413,generation=2,filelist=[_q9.cfe, _q9.cfs, _q9.si, _qa.cfe, _qa.cfs, _qa.si, _qb.cfe, _qb.cfs, _qb.si, _qc.cfe, _qc.cfs, _qc.si, _qd.cfe, _qd.cfs, _qd.si, _qe.cfe, _qe.cfs, _qe.si, _qf.cfe, _qf.cfs, _qf.si, _qg.cfe, _qg.cfs, _qg.si, _qh.cfe, _qh.cfs, _qh.si, _qi.cfe, _qi.cfs, _qi.si, _qj.cfe, _qj.cfs, _qj.si, _qk.cfe, _qk.cfs, _qk.si, _ql.cfe, _ql.cfs, _ql.si, _qm.cfe, _qm.cfs, _qm.si, _qn.cfe, _qn.cfs, _qn.si, _qo.cfe, _qo.cfs, _qo.si, _qp.cfe, _qp.cfs, _qp.si, _qq.cfe, _qq.cfs, _qq.si, _qr.cfe, _qr.cfs, _qr.si, _qs.cfe, _qs.cfs, _qs.si, _qt.cfe, _qt.cfs, _qt.si, _qu.cfe, _qu.cfs, _qu.si, _qv.cfe, _qv.cfs, _qv.si, _qw.cfe, _qw.cfs, _qw.si, _qx.cfe, _qx.cfs, _qx.si, _qy.cfe, _qy.cfs, _qy.si, _qz.cfe, _qz.cfs, _qz.si, _r0.cfe, _r0.cfs, _r0.si, _r1.cfe, _r1.cfs, _r1.si, _r2.cfe, _r2.cfs, _r2.si, _r3.cfe, _r3.cfs, _r3.si, _r4.cfe, _r4.cfs, _r4.si, _r5.cfe, _r5.cfs, _r5.si, _r6.cfe, _r6.cfs, _r6.si, _r7.cfe, _r7.cfs, _r7.si, _r8.cfe, _r8.cfs, _r8.si, _r9.cfe, _r9.cfs, _r9.si, _ra.cfe, _ra.cfs, _ra.si, _rb.cfe, _rb.cfs, _rb.si, _rc.cfe, _rc.cfs, _rc.si, _rd.cfe, _rd.cfs, _rd.si, _re.cfe, _re.cfs, _re.si, _rf.cfe, _rf.cfs, _rf.si, _rg.cfe, _rg.cfs, _rg.si, _rh.cfe, _rh.cfs, _rh.si, _ri.cfe, _ri.cfs, _ri.si, _rk.cfe, _rk.cfs, _rk.si, segments_2]}]> but was:<[{indexVersion=1426174267413,generation=3,filelist=[_ri.cfe, _ri.cfs, _ri.si, _rj.cfe, _rj.cfs, _rj.si, _rk.cfe, _rk.cfs, _rk.si, segments_3]}, {indexVersion=1426174267413,generation=2,filelist=[_q9.cfe, _q9.cfs, _q9.si, _qa.cfe, _qa.cfs, _qa.si, _qb.cfe, _qb.cfs, _qb.si, _qc.cfe, _qc.cfs, _qc.si, _qd.cfe, _qd.cfs, _qd.si, _qe.cfe, _qe.cfs, _qe.si, _qf.cfe, _qf.cfs, _qf.si, _qg.cfe, _qg.cfs, _qg.si, _qh.cfe, _qh.cfs, _qh.si, _qi.cfe, _qi.cfs, _qi.si, _qj.cfe, _qj.cfs, _qj.si, _qk.cfe, _qk.cfs, _qk.si, _ql.cfe, _ql.cfs, _ql.si, _qm.cfe, _qm.cfs, _qm.si, _qn.cfe, _qn.cfs, _qn.si, _qo.cfe, _qo.cfs, _qo.si, _qp.cfe, _qp.cfs, _qp.si, _qq.cfe, _qq.cfs, _qq.si, _qr.cfe, _qr.cfs, _qr.si, _qs.cfe, _qs.cfs, _qs.si, _qt.cfe, _qt.cfs, _qt.si, _qu.cfe, _qu.cfs, _qu.si, _qv.cfe, _qv.cfs, _qv.si, _qw.cfe, _qw.cfs, _qw.si, _qx.cfe, _qx.cfs, _qx.si, _qy.cfe, _qy.cfs, _qy.si, _qz.cfe, _qz.cfs, _qz.si, _r0.cfe, _r0.cfs, _r0.si, _r1.cfe, _r1.cfs, _r1.si, _r2.cfe, _r2.cfs, _r2.si, _r3.cfe, _r3.cfs, _r3.si, _r4.cfe, _r4.cfs, _r4.si, _r5.cfe, _r5.cfs, _r5.si, _r6.cfe, _r6.cfs, _r6.si, _r7.cfe, _r7.cfs, _r7.si, _r8.cfe, _r8.cfs, _r8.si, _r9.cfe, _r9.cfs, _r9.si, _ra.cfe, _ra.cfs, _ra.si, _rb.cfe, _rb.cfs, _rb.si, _rc.cfe, _rc.cfs, _rc.si, _rd.cfe, _rd.cfs, _rd.si, _re.cfe, _re.cfs, _re.si, _rf.cfe, _rf.cfs, _rf.si, _rg.cfe, _rg.cfs, _rg.si, _rh.cfe, _rh.cfs, _rh.si, _ri.cfe, _ri.cfs, _ri.si, _rk.cfe, _rk.cfs, _rk.si, segments_2]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1426174267413,generation=2,filelist=[_q9.cfe, _q9.cfs, _q9.si, _qa.cfe, _qa.cfs, _qa.si, _qb.cfe, _qb.cfs, _qb.si, _qc.cfe, _qc.cfs, _qc.si, _qd.cfe, _qd.cfs, _qd.si, _qe.cfe, _qe.cfs, _qe.si, _qf.cfe, _qf.cfs, _qf.si, _qg.cfe, _qg.cfs, _qg.si, _qh.cfe, _qh.cfs, _qh.si, _qi.cfe, _qi.cfs, _qi.si, _qj.cfe, _qj.cfs, _qj.si, _qk.cfe, _qk.cfs, _qk.si, _ql.cfe, _ql.cfs, _ql.si, _qm.cfe, _qm.cfs, _qm.si, _qn.cfe, _qn.cfs, _qn.si, _qo.cfe, _qo.cfs, _qo.si, _qp.cfe, _qp.cfs, _qp.si, _qq.cfe, _qq.cfs, _qq.si, _qr.cfe, _qr.cfs, _qr.si, _qs.cfe, _qs.cfs, _qs.si, _qt.cfe, _qt.cfs, _qt.si, _qu.cfe, _qu.cfs, _qu.si, _qv.cfe, _qv.cfs, _qv.si, _qw.cfe, _qw.cfs, _qw.si, _qx.cfe, _qx.cfs, _qx.si, _qy.cfe, _qy.cfs, _qy.si, _qz.cfe, _qz.cfs, _qz.si, _r0.cfe, _r0.cfs, _r0.si, _r1.cfe, _r1.cfs, _r1.si, _r2.cfe, _r2.cfs, _r2.si, _r3.cfe, _r3.cfs, _r3.si, _r4.cfe, _r4.cfs, _r4.si, _r5.cfe, _r5.cfs, _r5.si, _r6.cfe, _r6.cfs, _r6.si, _r7.cfe, _r7.cfs, _r7.si, _r8.cfe, _r8.cfs, _r8.si, _r9.cfe, _r9.cfs, _r9.si, _ra.cfe, _ra.cfs, _ra.si, _rb.cfe, _rb.cfs, _rb.si, _rc.cfe, _rc.cfs, _rc.si, _rd.cfe, _rd.cfs, _rd.si, _re.cfe, _re.cfs, _re.si, _rf.cfe, _rf.cfs, _rf.si, _rg.cfe, _rg.cfs, _rg.si, _rh.cfe, _rh.cfs, _rh.si, _ri.cfe, _ri.cfs, _ri.si, _rk.cfe, _rk.cfs, _rk.si, segments_2]}]> but was:<[{indexVersion=1426174267413,generation=3,filelist=[_ri.cfe, _ri.cfs, _ri.si, _rj.cfe, _rj.cfs, _rj.si, _rk.cfe, _rk.cfs, _rk.si, segments_3]}, {indexVersion=1426174267413,generation=2,filelist=[_q9.cfe, _q9.cfs, _q9.si, _qa.cfe, _qa.cfs, _qa.si, _qb.cfe, _qb.cfs, _qb.si, _qc.cfe, _qc.cfs, _qc.si, _qd.cfe, _qd.cfs, _qd.si, _qe.cfe, _qe.cfs, _qe.si, _qf.cfe, _qf.cfs, _qf.si, _qg.cfe, _qg.cfs, _qg.si, _qh.cfe, _qh.cfs, _qh.si, _qi.cfe, _qi.cfs, _qi.si, _qj.cfe, _qj.cfs, _qj.si, _qk.cfe, _qk.cfs, _qk.si, _ql.cfe, _ql.cfs, _ql.si, _qm.cfe, _qm.cfs, _qm.si, _qn.cfe, _qn.cfs, _qn.si, _qo.cfe, _qo.cfs, _qo.si, _qp.cfe, _qp.cfs, _qp.si, _qq.cfe, _qq.cfs, _qq.si, _qr.cfe, _qr.cfs, _qr.si, _qs.cfe, _qs.cfs, _qs.si, _qt.cfe, _qt.cfs, _qt.si, _qu.cfe, _qu.cfs, _qu.si, _qv.cfe, _qv.cfs, _qv.si, _qw.cfe, _qw.cfs, _qw.si, _qx.cfe, _qx.cfs, _qx.si, _qy.cfe, _qy.cfs, _qy.si, _qz.cfe, _qz.cfs, _qz.si, _r0.cfe, _r0.cfs, _r0.si, _r1.cfe, _r1.cfs, _r1.si, _r2.cfe, _r2.cfs, _r2.si, _r3.cfe, _r3.cfs, _r3.si, _r4.cfe, _r4.cfs, _r4.si, _r5.cfe, _r5.cfs, _r5.si, _r6.cfe, _r6.cfs, _r6.si, _r7.cfe, _r7.cfs, _r7.si, _r8.cfe, _r8.cfs, _r8.si, _r9.cfe, _r9.cfs, _r9.si, _ra.cfe, _ra.cfs, _ra.si, _rb.cfe, _rb.cfs, _rb.si, _rc.cfe, _rc.cfs, _rc.si, _rd.cfe, _rd.cfs, _rd.si, _re.cfe, _re.cfs, _re.si, _rf.cfe, _rf.cfs, _rf.si, _rg.cfe, _rg.cfs, _rg.si, _rh.cfe, _rh.cfs, _rh.si, _ri.cfe, _ri.cfs, _ri.si, _rk.cfe, _rk.cfs, _rk.si, segments_2]}]>
	at __randomizedtesting.SeedInfo.seed([6E6801B0270E8FC3:4BBF1A80574681C0]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1137)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds

Error Message:
soft529 wasn't fast enough

Stack Trace:
java.lang.AssertionError: soft529 wasn't fast enough
	at __randomizedtesting.SeedInfo.seed([6E6801B0270E8FC3:3FBCF830967DBF64]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds(SoftAutoCommitTest.java:111)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51865/c8n_1x2_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51865/c8n_1x2_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([6E6801B0270E8FC3:E63C3E6A89F2E23B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:51865/c8n_1x2_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33989/dti/nm/c8n_1x3_lf_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33989/dti/nm/c8n_1x3_lf_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([6E6801B0270E8FC3:E63C3E6A89F2E23B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33989/dti/nm/c8n_1x3_lf_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:11332/_ch/p/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11332/_ch/p/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([6E6801B0270E8FC3:E63C3E6A89F2E23B]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:62878/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:62878/collection1
	at __randomizedtesting.SeedInfo.seed([6E6801B0270E8FC3:E63C3E6A89F2E23B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:62878/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11792/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11792/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([6E6801B0270E8FC3:E63C3E6A89F2E23B]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11792/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 53 more




Build Log:
[...truncated 9831 lines...]
   [junit4] Suite: org.apache.solr.update.SoftAutoCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 6E6801B0270E8FC3-001/init-core-data-001
   [junit4]   2> 18752 T16 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 18755 T16 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> 18755 T16 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 18758 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 18759 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 18760 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 18973 T16 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 19100 T16 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 19244 T16 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 19246 T16 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 19345 T16 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 22264 T16 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 22380 T16 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 22385 T16 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 22653 T16 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 22687 T16 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 22693 T16 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 22702 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 22702 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 22703 T16 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 22704 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 22704 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 22723 T16 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 22724 T16 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr
   [junit4]   2> 22724 T16 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/'
   [junit4]   2> 22784 T16 oasc.CoreContainer.<init> New CoreContainer 8943451
   [junit4]   2> 22785 T16 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/]
   [junit4]   2> 22785 T16 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/lib
   [junit4]   2> 22786 T16 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/lib).
   [junit4]   2> 22803 T16 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 600000
   [junit4]   2> 22804 T16 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 22804 T16 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 60000
   [junit4]   2> 22805 T16 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 22805 T16 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 22805 T16 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 22806 T16 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 22806 T16 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 22807 T16 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 22807 T16 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 22808 T16 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 22809 T16 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=30000&connTimeout=30000&retry=true
   [junit4]   2> 22810 T16 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 22810 T16 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 22811 T16 oasc.CoreContainer.load Node Name: testNode
   [junit4]   1> CORE DESCRIPTOR: {transient=false, loadOnStartup=true, config=solrconfig.xml, collection=collection1, name=collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/, schema=schema.xml, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 6E6801B0270E8FC3-001/init-core-data-001, shard=shard1, instanceDir=collection1}
   [junit4]   2> 22814 T17 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 22817 T17 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 22818 T17 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 22820 T17 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 22986 T17 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 23046 T17 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 23126 T17 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 23128 T17 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 23205 T17 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 23667 T17 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 23727 T17 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 23730 T17 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 23758 T17 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 23765 T17 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 23770 T17 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 23771 T17 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 23771 T17 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 23772 T17 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 23773 T17 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 23773 T17 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 23774 T17 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 23774 T17 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 23775 T17 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 6E6801B0270E8FC3-001/init-core-data-001/
   [junit4]   2> 23783 T17 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 23783 T17 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 6E6801B0270E8FC3-001/init-core-data-001
   [junit4]   2> 23784 T17 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 6E6801B0270E8FC3-001/init-core-data-001/index/
   [junit4]   2> 23784 T17 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 6E6801B0270E8FC3-001/init-core-data-001/index' doesn't exist. Creating new index...
   [junit4]   2> 23836 T17 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 6E6801B0270E8FC3-001/init-core-data-001/index
   [junit4]   2> 23836 T17 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=18, maxMergeAtOnceExplicit=35, maxMergedSegmentMB=27.72265625, floorSegmentMB=1.4248046875, forceMergeDeletesPctAllowed=23.422596375358495, segmentsPerTier=44.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 23863 T17 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(SimpleFSDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 6E6801B0270E8FC3-001/index-SimpleFSDirectory-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@3b63ecdd),segFN=segments_1,generation=1}
   [junit4]   2> 23864 T17 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 24167 T17 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 24307 T17 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 24308 T17 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 24309 T17 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 24309 T17 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 24309 T17 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 24310 T17 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 24356 T17 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 24356 T17 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 24619 T17 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 24620 T17 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 24620 T17 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 24649 T17 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 24652 T17 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 24654 T17 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 24657 T17 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 24920 T17 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 25042 T17 oasc.RequestHandlers.initHandlersFromConfig Registered paths: spellCheckCompRH_Direct,defaults,/admin/threads,/mlt,/update,/search-facet-def,/debug/dump,/replication,/update/csv,/search-facet-invariants,/admin/plugins,lazy,mltrh,/admin/luke,/schema,spellCheckCompRH,spellCheckCompRH1,/admin/logging,/admin/file,/admin/properties,spellCheckWithWordbreak,dismax,/config,/update/json,tvrh,/update/json/docs,/get,spellCheckWithWordbreak_Direct,/admin/segments,/terms,mock,standard,/admin/mbeans,/admin/system,/admin/ping
   [junit4]   2> 25052 T17 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 25054 T17 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 25131 T17 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 25132 T17 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 25133 T17 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=14, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.251924232805395]
   [junit4]   2> 25156 T17 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(SimpleFSDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 6E6801B0270E8FC3-001/index-SimpleFSDirectory-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@3b63ecdd),segFN=segments_1,generation=1}
   [junit4]   2> 25157 T17 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 25157 T17 oass.SolrIndexSearcher.<init> Opening Searcher@1b5d3e3a[collection1] main
   [junit4]   2> 25173 T17 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 25174 T17 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 25174 T17 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 25174 T17 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 25524 T17 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 25922 T17 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 25924 T18 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 25925 T17 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 25928 T18 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 25929 T18 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 25929 T18 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 25932 T18 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 25935 T18 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 25936 T18 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 25938 T18 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 25940 T18 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 25941 T18 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@1b5d3e3a[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 25958 T16 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 25972 T16 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeMixedAdds
   [junit4]   2> 25973 T16 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 25975 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 25976 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 25977 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 26184 T16 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 26234 T16 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 26337 T16 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 26338 T16 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 26436 T16 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 26944 T16 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 26968 T16 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 26971 T16 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 27013 T16 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 27019 T16 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 27024 T16 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 27026 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 27026 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 27027 T16 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 27028 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 27028 T16 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 27029 T16 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 27029 T16 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 6E6801B0270E8FC3-001/init-core-data-001/
   [junit4]   2> 27030 T16 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 27031 T16 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 6E6801B0270E8FC3-001/init-core-data-001/index/
   [junit4]   2> 27035 T16 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 27035 T16 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 27036 T16 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 27036 T16 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 27037 T16 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 27037 T16 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 27038 T16 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 27039 T16 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 27039 T16 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 27040 T16 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 27041 T16 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 27041 T16 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 27058 T16 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 27061 T16 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 27063 T16 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 27065 T16 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 27068 T16 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 27070 T16 oasc.RequestHandlers.initHandlersFromConfig Registered paths: spellCheckCompRH_Direct,defaults,/admin/threads,/mlt,/update,/search-facet-def,/debug/dump,/replication,/update/csv,/search-facet-invariants,/admin/plugins,lazy,mltrh,/admin/luke,/schema,spellCheckCompRH,spellCheckCompRH1,/admin/logging,/admin/file,/admin/properties,spellCheckWithWordbreak,dismax,/config,/update/json,tvrh,/update/json/docs,/get,spellCheckWithWordbreak_Direct,/admin/segments,/terms,mock,standard,/admin/mbeans,/admin/system,/admin/ping
   [junit4]   2> 27072 T16 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 27074 T16 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 27080 T16 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 27080 T16 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 27081 T16 oass.SolrIndexSearcher.<init> Opening Searcher@2d9917bc[collection1] main
   [junit4]   2> 27082 T16 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 27083 T16 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 27083 T16 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 27084 T16 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 27086 T16 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 27099 T16 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 27100 T20 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 27101 T16 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 27102 T16 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 27102 T16 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 27102 T20 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 27103 T20 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 27104 T20 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 27105 T20 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 27107 T20 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 27109 T20 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 27109 T16 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@6677a632
   [junit4]   2> 27111 T20 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 27112 T20 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 27112 T16 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(SimpleFSDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 6E6801B0270E8FC3-001/index-SimpleFSDirectory-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@3b63ecdd),segFN=segments_1,generation=1}
   [junit4]   2> 27113 T20 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@2d9917bc[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 27113 T16 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 27115 T16 oasu.DefaultSolrCoreState.newIndexWriter New IndexWriter is ready to be used.
   [junit4]   2> 27115 T16 oass.SolrIndexSearcher.<init> Opening Searcher@63504207[collection1] main
   [junit4]   2> 27130 T16 oasc.CoreContainer.registerCore replacing core: collection1
   [junit4]   2> 27131 T16 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@47262043
   [junit4]   2> 27131 T16 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 27132 T16 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 27132 T20 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@63504207[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> ASYNC  NEW_CORE C1 name=collection1 org.apache.solr.core.SolrCore@189e8b21
   [junit4]   2> 27191 T16 C1 oasup.LogUpdateProcessor.finish [collection1] webapp=null path=null params={} {add=[529]} 0 57
   [junit4]   2> 27692 T22 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=true,prepareCommit=false}
   [junit4]   2> 28216 T16 oas.SolrTestCaseJ4.tearDown ###Ending testSoftAndHardCommitMaxTimeMixedAdds
   [junit4]   2> 28227 T22 oass.SolrIndexSearcher.<init> Opening Searcher@5286530e[collection1] main
   [junit4]   2> 28228 T22 oasu.DirectUpdateHandler2.commit end_commit_flush
   [junit4]   2> 28228 T20 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@5286530e[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader(Uninverting(_0(5.1.0):C1)))}
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=SoftAutoCommitTest -Dtests.method=testSoftAndHardCommitMaxTimeMixedAdds -Dtests.seed=6E6801B0270E8FC3 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=ru_RU -Dtests.timezone=Europe/Tirane -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 2.33s J3 | SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: soft529 wasn't fast enough
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([6E6801B0270E8FC3:3FBCF830967DBF64]:0)
   [junit4]    > 	at org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds(SoftAutoCommitTest.java:111)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 28298 T16 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeDelete
   [junit4]   2> 28299 T16 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 28301 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 28302 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 28302 T16 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 28391 T21 oasu.DirectUpdateHandler2.commit start commit{,optimize=false,openSearcher=true,waitSearcher=true,expungeDeletes=false,softCommit=false,prepareCommit=false}
   [junit4]   2> 28418 T21 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=2
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(SimpleFSDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 6E6801B0270E8FC3-001/index-SimpleFSDirectory-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@3b63ecdd),segFN=segments_1,generation=1}
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(SimpleFSDirectory@/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 6E6801B0270E8FC3-001/index-SimpleFSDirectory-001 lockFactory=org.apache.lucene.store.NativeFSLockFactory@3b63ecdd),segFN=segments_2,generation=2}
   [junit4]   2> 28419 T21 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 2
   [junit4]   2> 28422 T21 oasc.SolrCore.openNewSearcher SolrIndexSearcher has not changed - not re-opening: org.apache.solr.search.SolrIn

[...truncated too long message...]

osing 0 connections to: http://127.0.0.1:11781/
   [junit4]   2> 2278453 T9658 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:11776/
   [junit4]   2> 2278453 T9658 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:11800/
   [junit4]   2> 2278453 T9658 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:11792/
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=6E6801B0270E8FC3 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=ar_SY -Dtests.timezone=EAT -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   70.9s J0 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11792/repfacttest_c8n_1x3_shard1_replica2
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([6E6801B0270E8FC3:E63C3E6A89F2E23B]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:11792/repfacttest_c8n_1x3_shard1_replica2
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	... 53 more
   [junit4]   2> 2278461 T9658 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 6E6801B0270E8FC3-001
   [junit4]   2> 70939 T9657 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=DefaultSimilarity, locale=ar_SY, timezone=EAT
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=61253536,total=344457216
   [junit4]   2> NOTE: All tests run in this JVM: [TestHdfsUpdateLog, TestUpdate, TestSolrIndexConfig, DistribJoinFromCollectionTest, DistributedQueryElevationComponentTest, TestFieldTypeResource, TestSolrDeletionPolicy2, TestPartialUpdateDeduplication, HdfsChaosMonkeySafeLeaderTest, CoreMergeIndexesAdminHandlerTest, DebugComponentTest, MBeansHandlerTest, DistributedQueueTest, RegexBoostProcessorTest, TestFieldTypeCollectionResource, MinimalSchemaTest, TestReplicationHandlerBackup, TestTrackingShardHandlerFactory, TestFieldCollectionResource, PrimitiveFieldTypeTest, TestCursorMarkWithoutUniqueKey, TestSchemaVersionResource, QueryParsingTest, TimeZoneUtilsTest, RequiredFieldsTest, ShardRoutingCustomTest, ExternalFileFieldSortTest, StatelessScriptUpdateProcessorFactoryTest, TestTrie, TestManagedStopFilterFactory, TestMissingGroups, TestCharFilters, TestManagedResourceStorage, HdfsSyncSliceTest, UUIDFieldTest, SolrCmdDistributorTest, HdfsBasicDistributedZk2Test, TestBlobHandler, TestXIncludeConfig, CloudMLTQParserTest, IndexBasedSpellCheckerTest, TestMiniSolrCloudClusterSSL, TestDFRSimilarityFactory, TestStressReorder, SpatialHeatmapFacetsTest, CustomCollectionTest, DirectSolrSpellCheckerTest, TestRebalanceLeaders, PolyFieldTest, TestCoreContainer, SegmentsInfoRequestHandlerTest, DistanceFunctionTest, TestSearcherReuse, HttpPartitionTest, DistributedFacetPivotLongTailTest, ZkNodePropsTest, DistributedSpellCheckComponentTest, BasicFunctionalityTest, BasicDistributedZk2Test, UnloadDistributedZkTest, OpenCloseCoreStressTest, ZkSolrClientTest, TestRangeQuery, SimpleFacetsTest, TestSolr4Spatial, SolrCoreTest, QueryElevationComponentTest, BadIndexSchemaTest, TestSort, TestBadConfig, ShowFileRequestHandlerTest, SignatureUpdateProcessorFactoryTest, SuggesterFSTTest, SuggesterTest, TestAtomicUpdateErrorCases, TestRemoteStreaming, SolrInfoMBeanTest, LukeRequestHandlerTest, FileBasedSpellCheckerTest, DocumentBuilderTest, MoreLikeThisHandlerTest, FastVectorHighlighterTest, SolrPluginUtilsTest, TestComponentsName, TestLFUCache, UpdateParamsTest, ResponseLogComponentTest, SampleTest, TestConfig, SpellPossibilityIteratorTest, TestIBSimilarityFactory, ScriptEngineTest, URLClassifyProcessorTest, TestSuggestSpellingConverter, DateFieldTest, DistributedIntervalFacetingTest, DeleteReplicaTest, ExternalCollectionsTest, LeaderFailoverAfterPartitionTest, OverseerStatusTest, ReplicationFactorTest]
   [junit4] Completed on J0 in 71.22s, 1 test, 1 error <<< FAILURES!

[...truncated 297 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 472 suites, 1883 tests, 5 errors, 3 failures, 58 ignored (9 assumptions)

Total time: 144 minutes 37 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 754423287 bytes
Compression is 0.0%
Took 2 min 24 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 783 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/783/

8 tests failed.
REGRESSION:  org.apache.solr.cloud.DistribDocExpirationUpdateProcessorTest.test

Error Message:
There are still nodes recoverying - waited for 30 seconds

Stack Trace:
java.lang.AssertionError: There are still nodes recoverying - waited for 30 seconds
	at __randomizedtesting.SeedInfo.seed([28732153B03FC6E8:A0271E891EC3AB10]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractDistribZkTestBase.waitForRecoveriesToFinish(AbstractDistribZkTestBase.java:174)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.java:875)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.waitForThingsToLevelOut(AbstractFullDistribZkTestBase.java:1490)
	at org.apache.solr.cloud.DistribDocExpirationUpdateProcessorTest.test(DistribDocExpirationUpdateProcessorTest.java:70)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.MultiThreadedOCPTest

Error Message:
1 thread leaked from SUITE scope at org.apache.solr.cloud.MultiThreadedOCPTest:     1) Thread[id=1280, name=OverseerThreadFactory-522-thread-5, state=TIMED_WAITING, group=Overseer collection creation process.]         at java.lang.Thread.sleep(Native Method)         at org.apache.solr.cloud.OverseerCollectionProcessor.waitForCoreNodeName(OverseerCollectionProcessor.java:1825)         at org.apache.solr.cloud.OverseerCollectionProcessor.splitShard(OverseerCollectionProcessor.java:1712)         at org.apache.solr.cloud.OverseerCollectionProcessor.processMessage(OverseerCollectionProcessor.java:624)         at org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2863)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 1 thread leaked from SUITE scope at org.apache.solr.cloud.MultiThreadedOCPTest: 
   1) Thread[id=1280, name=OverseerThreadFactory-522-thread-5, state=TIMED_WAITING, group=Overseer collection creation process.]
        at java.lang.Thread.sleep(Native Method)
        at org.apache.solr.cloud.OverseerCollectionProcessor.waitForCoreNodeName(OverseerCollectionProcessor.java:1825)
        at org.apache.solr.cloud.OverseerCollectionProcessor.splitShard(OverseerCollectionProcessor.java:1712)
        at org.apache.solr.cloud.OverseerCollectionProcessor.processMessage(OverseerCollectionProcessor.java:624)
        at org.apache.solr.cloud.OverseerCollectionProcessor$Runner.run(OverseerCollectionProcessor.java:2863)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([28732153B03FC6E8]:0)


FAILED:  org.apache.solr.handler.FieldAnalysisRequestHandlerTest.testSpatial

Error Message:
expected:<s[]> but was:<s[vk6+]>

Stack Trace:
org.junit.ComparisonFailure: expected:<s[]> but was:<s[vk6+]>
	at __randomizedtesting.SeedInfo.seed([28732153B03FC6E8:8C7004CEB23FB061]:0)
	at org.junit.Assert.assertEquals(Assert.java:125)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.FieldAnalysisRequestHandlerTest.testSpatial(FieldAnalysisRequestHandlerTest.java:426)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:39758/g_ivp/c8n_1x2_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:39758/g_ivp/c8n_1x2_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([28732153B03FC6E8:A0271E891EC3AB10]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:39758/g_ivp/c8n_1x2_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:45798/c8n_1x3_lf_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:45798/c8n_1x3_lf_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([28732153B03FC6E8:A0271E891EC3AB10]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:45798/c8n_1x3_lf_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:31637/c_/mt/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:31637/c_/mt/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([28732153B03FC6E8:A0271E891EC3AB10]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33807/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33807/collection1
	at __randomizedtesting.SeedInfo.seed([28732153B03FC6E8:A0271E891EC3AB10]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33807/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33933/repfacttest_c8n_1x3_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33933/repfacttest_c8n_1x3_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([28732153B03FC6E8:A0271E891EC3AB10]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33933/repfacttest_c8n_1x3_shard1_replica3
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 53 more




Build Log:
[...truncated 9979 lines...]
   [junit4] Suite: org.apache.solr.handler.FieldAnalysisRequestHandlerTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.FieldAnalysisRequestHandlerTest 28732153B03FC6E8-001/init-core-data-001
   [junit4]   2> 88178 T384 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (true)
   [junit4]   2> 88181 T384 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> 88181 T384 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 88183 T384 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 88186 T384 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 88187 T384 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 88335 T384 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 88386 T384 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 88501 T384 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 88503 T384 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 88524 T384 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 89186 T384 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 89209 T384 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 89213 T384 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 89240 T384 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 89247 T384 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 89252 T384 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 89253 T384 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 89254 T384 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 89255 T384 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 89255 T384 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 89256 T384 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 89256 T384 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 89257 T384 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr
   [junit4]   2> 89257 T384 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/'
   [junit4]   2> 89275 T384 oasc.CoreContainer.<init> New CoreContainer 1111078617
   [junit4]   2> 89276 T384 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/]
   [junit4]   2> 89276 T384 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/lib
   [junit4]   2> 89277 T384 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/lib).
   [junit4]   2> 89290 T384 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 600000
   [junit4]   2> 89291 T384 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 89291 T384 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 60000
   [junit4]   2> 89292 T384 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 89292 T384 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 89292 T384 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 89293 T384 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 89293 T384 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 89294 T384 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 89294 T384 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 89294 T384 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 89295 T384 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=30000&connTimeout=30000&retry=true
   [junit4]   2> 89296 T384 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 89296 T384 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 89296 T384 oasc.CoreContainer.load Node Name: testNode
   [junit4]   1> CORE DESCRIPTOR: {loadOnStartup=true, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/, name=collection1, schema=schema.xml, shard=shard1, instanceDir=collection1, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.FieldAnalysisRequestHandlerTest 28732153B03FC6E8-001/init-core-data-001, collection=collection1, transient=false}
   [junit4]   2> 89300 T385 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 89302 T385 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 89302 T385 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 89303 T385 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 89337 T385 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 89372 T385 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 89449 T385 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 89450 T385 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 89465 T385 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 89911 T385 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 89926 T385 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 89930 T385 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 89954 T385 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 89960 T385 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 89965 T385 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 89966 T385 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 89966 T385 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 89967 T385 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 89968 T385 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 89968 T385 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 89968 T385 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 89969 T385 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 89970 T385 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.FieldAnalysisRequestHandlerTest 28732153B03FC6E8-001/init-core-data-001/
   [junit4]   2> 89970 T385 oasc.JmxMonitoredMap.<init> No JMX servers found, not exposing Solr information with JMX.
   [junit4]   2> 89971 T385 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.FieldAnalysisRequestHandlerTest 28732153B03FC6E8-001/init-core-data-001
   [junit4]   2> 89971 T385 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.FieldAnalysisRequestHandlerTest 28732153B03FC6E8-001/init-core-data-001/index/
   [junit4]   2> 89972 T385 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.FieldAnalysisRequestHandlerTest 28732153B03FC6E8-001/init-core-data-001/index' doesn't exist. Creating new index...
   [junit4]   2> 89972 T385 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.FieldAnalysisRequestHandlerTest 28732153B03FC6E8-001/init-core-data-001/index
   [junit4]   2> 89973 T385 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=22, maxMergeAtOnceExplicit=39, maxMergedSegmentMB=38.20703125, floorSegmentMB=1.7021484375, forceMergeDeletesPctAllowed=19.875282540524385, segmentsPerTier=50.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 89974 T385 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@a8f86b3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@54d4572f),segFN=segments_1,generation=1}
   [junit4]   2> 89974 T385 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 89979 T385 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 89980 T385 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 89980 T385 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 89980 T385 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 89981 T385 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 89981 T385 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 89981 T385 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 89982 T385 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 89983 T385 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 89984 T385 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 89984 T385 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 89984 T385 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 90013 T385 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 90016 T385 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 90018 T385 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 90021 T385 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 90025 T385 oasc.PluginBag.createPlugin requestHandler : 'lazy' created with startup=lazy 
   [junit4]   2> 90027 T385 oasc.RequestHandlers.initHandlersFromConfig Registered paths: spellCheckCompRH_Direct,/admin/segments,/get,/search-facet-def,/update/json/docs,defaults,standard,/mlt,lazy,/admin/properties,/search-facet-invariants,/admin/plugins,mock,/admin/luke,/update/csv,/admin/ping,/admin/mbeans,/schema,mltrh,/update,/admin/system,/debug/dump,spellCheckCompRH,/admin/logging,/admin/file,spellCheckWithWordbreak,/replication,spellCheckWithWordbreak_Direct,/config,spellCheckCompRH1,dismax,tvrh,/update/json,/admin/threads,/terms
   [junit4]   2> 90029 T385 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 90031 T385 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 90032 T385 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 90032 T385 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 90034 T385 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=2018240725, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 90035 T385 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@a8f86b3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@54d4572f),segFN=segments_1,generation=1}
   [junit4]   2> 90036 T385 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 90036 T385 oass.SolrIndexSearcher.<init> Opening Searcher@249ca61f[collection1] main
   [junit4]   2> 90037 T385 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 90038 T385 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 90038 T385 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 90038 T385 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 90079 T385 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 90146 T385 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 90147 T386 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 90147 T385 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 90148 T386 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 90148 T386 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 90149 T386 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 90150 T384 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 90150 T386 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 90151 T386 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 90153 T386 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 90154 T386 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 90155 T386 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 90155 T386 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@249ca61f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 90156 T384 oas.SolrTestCaseJ4.setUp ###Starting testResolveAnalysisRequest
   [junit4]   2> 90194 T384 oas.SolrTestCaseJ4.tearDown ###Ending testResolveAnalysisRequest
   [junit4]   2> 90201 T384 oas.SolrTestCaseJ4.setUp ###Starting testCharFilterAnalysis
   [junit4]   2> 90307 T384 oas.SolrTestCaseJ4.tearDown ###Ending testCharFilterAnalysis
   [junit4]   2> 90314 T384 oas.SolrTestCaseJ4.setUp ###Starting testSpatial
   [junit4]   2> 90452 T384 oass.AbstractSpatialPrefixTreeFieldType.newSpatialStrategy location_rpt{class=org.apache.solr.schema.SpatialRecursivePrefixTreeFieldType,analyzer=org.apache.solr.schema.FieldType$DefaultAnalyzer,args={geo=true, class=solr.SpatialRecursivePrefixTreeFieldType, distanceUnits=kilometers, distErrPct=0.025, maxDistErr=0.001}} strat: RecursivePrefixTreeStrategy(SPG:(GeohashPrefixTree(maxLevels:11,ctx:SpatialContext{geo=true, calculator=Haversine, worldBounds=Rect(minX=-180.0,maxX=180.0,minY=-90.0,maxY=90.0)})),pruneLeafyBranches) maxLevels: 11
   [junit4]   2> 90535 T384 oas.SolrTestCaseJ4.tearDown ###Ending testSpatial
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=FieldAnalysisRequestHandlerTest -Dtests.method=testSpatial -Dtests.seed=28732153B03FC6E8 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=ar_TN -Dtests.timezone=Africa/Asmara -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 0.30s J1 | FieldAnalysisRequestHandlerTest.testSpatial <<<
   [junit4]    > Throwable #1: org.junit.ComparisonFailure: expected:<s[]> but was:<s[vk6+]>
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([28732153B03FC6E8:8C7004CEB23FB061]:0)
   [junit4]    > 	at org.apache.solr.handler.FieldAnalysisRequestHandlerTest.testSpatial(FieldAnalysisRequestHandlerTest.java:426)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 90616 T384 oas.SolrTestCaseJ4.setUp ###Starting testHandleAnalysisRequest
   [junit4]   2> 90624 T384 oas.SolrTestCaseJ4.tearDown ###Ending testHandleAnalysisRequest
   [junit4]   2> 90631 T384 oas.SolrTestCaseJ4.setUp ###Starting testPositionHistoryWithWDF
   [junit4]   2> 90658 T384 oas.SolrTestCaseJ4.tearDown ###Ending testPositionHistoryWithWDF
   [junit4]   2> 90659 T384 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> 90659 T384 oasc.CoreContainer.shutdown Shutting down CoreContainer instance=1111078617
   [junit4]   2> 90659 T384 oasc.SolrCore.close [collection1]  CLOSING SolrCore org.apache.solr.core.SolrCore@1d634bdd
   [junit4]   2> 90660 T384 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=0,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=0,cumulative_deletesById=0,cumulative_deletesByQuery=0,cumulative_errors=0}
   [junit4]   2> 90660 T384 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 90660 T384 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 90660 T384 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 90662 T384 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 90662 T384 oasc.CachingDirectoryFactory.close Closing MockDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 90662 T384 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.FieldAnalysisRequestHandlerTest 28732153B03FC6E8-001/init-core-data-001/index [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.FieldAnalysisRequestHandlerTest 28732153B03FC6E8-001/init-core-data-001/index;done=false>>]
   [junit4]   2> 90663 T384 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.FieldAnalysisRequestHandlerTest 28732153B03FC6E8-001/init-core-data-001/index
   [junit4]   2> 90663 T384 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.FieldAnalysisRequestHandlerTest 28732153B03FC6E8-001/init-core-data-001 [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.FieldAnalysisRequestHandlerTest 28732153B03FC6E8-001/init-core-data-001;done=false>>]
   [junit4]   2> 90663 T384 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.FieldAnalysisRequestHandlerTest 28732153B03FC6E8-001/init-core-data-001
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.handler.FieldAnalysisRequestHandlerTest 28732153B03FC6E8-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=DefaultSimilarity, locale=ar_TN, timezone=Africa/Asmara
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=126482768,total=318767104
   [junit4]   2> NOTE: All tests run in this JVM: [DistributedQueryComponentCustomSortTest, TestSearchPerf, DistributedTermsComponentTest, TestConfigOverlay, DirectSolrConnectionTest, FieldAnalysisRequestHandlerTest]
   [junit4] Completed on J1 in 2.53s, 5 tests, 1 failure <<< FAILURES!

[...truncated 151 lines...]
   [junit4] Suite: org.apache.solr.cloud.HttpPartitionTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 28732153B03FC6E8-001/init-core-data-001
   [junit4]   2> 229693 T957 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /g_ivp/
   [junit4]   2> 229704 T957 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 229706 T958 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 229806 T957 oasc.ZkTestServer.run start zk server on port:39755
   [junit4]   2> 229807 T957 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 229809 T957 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 229814 T965 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@142ce214 name:ZooKeeperConnection Watcher:127.0.0.1:39755 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 229815 T957 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 229815 T957 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 229816 T957 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 229818 T957 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 229819 T957 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 229820 T968 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@73a31e0a name:ZooKeeperConnection Watcher:127.0.0.1:39755/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 229821 T957 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 229821 T957 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 229821 T957 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 229825 T957 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 229827 T957 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 229828 T957 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 229831 T957 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 229832 T957 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 229835 T957 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 229836 T957 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 229838 T957 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 229839 T957 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 229841 T957 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 229842 T957 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 229844 T957 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 229844 T957 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 229846 T957 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 229847 T957 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 229849 T957 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 229849 T957 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 229851 T957 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 229852 T957 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 229854 T957 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 229854 T957 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 229857 T957 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 229857 T957 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 229885 T957 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 229885 T957 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 231285 T957 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 28732153B03FC6E8-001/control-001/cores/collection1
   [junit4]   2> 231289 T957 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 231292 T957 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:39759
   [junit4]   2> 231293 T957 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/g_ivp, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 28732153B03FC6E8-001/tempDir-001/control/data, hostPort=39758, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 28732153B03FC6E8-001/control-001/cores}
   [junit4]   2> 231294 T957 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 231294 T957 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 28732153B03FC6E8-001/control-001/'
   [junit4]   2> 231336 T957 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 28732153B03FC6E8-001/control-001/solr.xml
   [junit4]   2> 231378 T957 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 28732153B03FC6E8-001/control-001/cores
   [junit4]   2> 231379 T957 oasc.CoreContainer.<init> New CoreContainer 578093986
   [junit4]   2> 231379 T957 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 28732153B03FC6E8-001/control-001/]
   [junit4]   2> 231379 T957 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 28732153B03FC6E8-001/control-001/lib
   [junit4]   2> 231380 T957 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 28732153B03FC6E8-001/control-001/lib).
   [junit4]   2> 231389 T957 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 231390 T957 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 231390 T957 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 231391 T957 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 231391 T957 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 231391 T957 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 231392 T957 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 231392 T957 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 231392 T957 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 231392 T957 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 231393 T957 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 231393 T957 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 231394 T957 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 231394 T957 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 231394 T957 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 231395 T957 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:39755/solr
   [junit4]   2> 231395 T957 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 231395 T957 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 231396 T957 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 231408 T980 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@229b88c6 name:ZooKeeperConnection Watcher:127.0.0.1:39755 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 231409 T957 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 231410 T957 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 231413 T957 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 231414 T983 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@452d87e3 name:ZooKeeperConnection Watcher:127.0.0.1:39755/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 231414 T957 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 231417 T957 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 231419 T957 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 231422 T957 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 231425 T957 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 231427 T957 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 231430 T957 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 231433 T957 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 231434 T957 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 231436 T957 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:39758_g_ivp
   [junit4]   2> 231437 T957 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:39758_g_ivp
   [junit4]   2> 231439 T957 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 231441 T957 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 231442 T957 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 231444 T957 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:39758_g_ivp
   [junit4]   2> 231445 T957 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 231446 T957 oasc.Overseer.start Overseer (id=93460051996901379-127.0.0.1:39758_g_ivp-n_0000000000) starting
   [junit4]   2> 231449 T957 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 231455 T957 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 231456 T985 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 231456 T957 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 231457 T984 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 231461 T957 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 28732153B03FC6E8-001/control-001/cores
   [junit4]   1> CORE DESCRIPTOR: {name=collection1, loadOnStartup=true, transient=false, dataDir=data/, coreNodeName=, collection=control_collection, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 28732153B03FC6E8-001/control-001/cores/collection1/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 28732153B03FC6E8-001/control-001/cores/collection1, config=solrconfig.xml, schema=schema.xml, shard=}
   [junit4]   2> 231465 T957 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 28732153B03FC6E8-001/control-001/cores/collection1/
   [junit4]   2> 231465 T957 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 231466 T987 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 231467 T987 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 231468 T983 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 231468 T987 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 231469 T984 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:39758_g_ivp",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39758/g_ivp",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null} current state version: 0
   [junit4]   2> 231470 T984 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "node_name":"127.0.0.1:39758_g_ivp",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:39758/g_ivp",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null

[...truncated too long message...]

asc.SocketProxy.close WARN Closing 1 connections to: http://127.0.0.1:33923/
   [junit4]   2> 2107035 T5760 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:33958/
   [junit4]   2> 2107035 T5760 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:33941/
   [junit4]   2> 2107036 T5760 oasc.SocketProxy.close WARN Closing 10 connections to: http://127.0.0.1:33933/
   [junit4]   2> 2107036 T5760 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 2107036 T5760 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:33923/
   [junit4]   2> 2107037 T5760 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:33958/
   [junit4]   2> 2107037 T5760 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:33941/
   [junit4]   2> 2107037 T5760 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:33933/
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=28732153B03FC6E8 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=el -Dtests.timezone=America/Resolute -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   71.4s J0 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33933/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([28732153B03FC6E8:A0271E891EC3AB10]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33933/repfacttest_c8n_1x3_shard1_replica3
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	... 53 more
   [junit4]   2> 2107059 T5760 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 28732153B03FC6E8-001
   [junit4]   2> 71358 T5759 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=DummyCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=DUMMY, chunkSize=8, maxDocsPerChunk=365, blockSize=843), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=DUMMY, chunkSize=8, blockSize=843)), sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=el, timezone=America/Resolute
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=146556072,total=333971456
   [junit4]   2> NOTE: All tests run in this JVM: [DistributedQueryElevationComponentTest, TestSearcherReuse, SpatialHeatmapFacetsTest, TestCoreContainer, SuggesterTest, TestSolrIndexConfig, TestUniqueKeyFieldResource, TestSort, TimeZoneUtilsTest, TestRebalanceLeaders, SimpleFacetsTest, DistributedQueueTest, HdfsCollectionsAPIDistributedZkTest, HttpPartitionTest, ExternalFileFieldSortTest, IndexBasedSpellCheckerTest, MBeansHandlerTest, HighlighterConfigTest, SolrCmdDistributorTest, ParsingFieldUpdateProcessorsTest, ShowFileRequestHandlerTest, TestAtomicUpdateErrorCases, MinimalSchemaTest, UUIDFieldTest, TestIBSimilarityFactory, TestSolrJ, RecoveryZkTest, TestFieldTypeCollectionResource, CoreMergeIndexesAdminHandlerTest, CustomCollectionTest, SolrTestCaseJ4Test, LeaderInitiatedRecoveryOnCommitTest, DistributedFacetPivotLongTailTest, DocExpirationUpdateProcessorFactoryTest, DOMUtilTest, BasicFunctionalityTest, TestRawResponseWriter, TestDynamicLoading, HdfsSyncSliceTest, CloudMLTQParserTest, TestCharFilters, SegmentsInfoRequestHandlerTest, RequestHandlersTest, RegexBoostProcessorTest, DirectSolrSpellCheckerTest, BasicDistributedZkTest, TestReplicationHandlerBackup, OverseerCollectionProcessorTest, TestFastLRUCache, TestFaceting, HdfsBasicDistributedZk2Test, ZkStateWriterTest, OverseerTest, TestHdfsUpdateLog, TestSolrDeletionPolicy2, TestObjectReleaseTracker, TestTrie, BasicDistributedZk2Test, ShardRoutingTest, TestReplicationHandler, OverseerStatusTest, ReplicationFactorTest]
   [junit4] Completed on J0 in 71.90s, 1 test, 1 error <<< FAILURES!

[...truncated 335 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 472 suites, 1882 tests, 1 suite-level error, 5 errors, 2 failures, 58 ignored (9 assumptions)

Total time: 143 minutes 57 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 731935788 bytes
Compression is 0.0%
Took 2 min 13 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 782 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/782/

5 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57192/c8n_1x2_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57192/c8n_1x2_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([44060DA38D0FC4E9:CC52327923F3A911]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:57192/c8n_1x2_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:42967/c8n_1x3_lf_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:42967/c8n_1x3_lf_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([44060DA38D0FC4E9:CC52327923F3A911]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:42967/c8n_1x3_lf_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:35426/mah/sn/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:35426/mah/sn/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([44060DA38D0FC4E9:CC52327923F3A911]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:41958/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:41958/collection1
	at __randomizedtesting.SeedInfo.seed([44060DA38D0FC4E9:CC52327923F3A911]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:41958/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18962/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18962/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([44060DA38D0FC4E9:CC52327923F3A911]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18962/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 53 more




Build Log:
[...truncated 10238 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/init-core-data-001
   [junit4]   2> 233748 T1485 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 233755 T1485 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 233756 T1486 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 233856 T1485 oasc.ZkTestServer.run start zk server on port:41927
   [junit4]   2> 233857 T1485 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 233858 T1485 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 233862 T1493 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1bd25832 name:ZooKeeperConnection Watcher:127.0.0.1:41927 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 233862 T1485 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 233863 T1485 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 233863 T1485 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 233866 T1485 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 233867 T1485 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 233869 T1496 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@c80c65e name:ZooKeeperConnection Watcher:127.0.0.1:41927/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 233869 T1485 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 233870 T1485 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 233871 T1485 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 233873 T1485 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 233875 T1485 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 233876 T1485 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 233879 T1485 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 233879 T1485 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 233882 T1485 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 233883 T1485 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 233885 T1485 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 233886 T1485 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 233888 T1485 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 233889 T1485 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 233890 T1485 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 233891 T1485 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 233893 T1485 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 233894 T1485 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 233896 T1485 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 233896 T1485 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 233898 T1485 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 233899 T1485 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 233901 T1485 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 233901 T1485 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 233903 T1485 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 233904 T1485 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 233906 T1485 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 233906 T1485 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 235438 T1485 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/control-001/cores/collection1
   [junit4]   2> 235442 T1485 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 235445 T1485 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:41938
   [junit4]   2> 235446 T1485 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, hostPort=41937, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/tempDir-001/control/data}
   [junit4]   2> 235446 T1485 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 235446 T1485 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/control-001/'
   [junit4]   2> 235480 T1485 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/control-001/solr.xml
   [junit4]   2> 235519 T1485 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/control-001/cores
   [junit4]   2> 235519 T1485 oasc.CoreContainer.<init> New CoreContainer 1481314725
   [junit4]   2> 235520 T1485 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/control-001/]
   [junit4]   2> 235520 T1485 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/control-001/lib
   [junit4]   2> 235521 T1485 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/control-001/lib).
   [junit4]   2> 235530 T1485 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 235530 T1485 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 235530 T1485 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 235530 T1485 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 235530 T1485 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 235531 T1485 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 235531 T1485 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 235531 T1485 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 235531 T1485 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 235531 T1485 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 235532 T1485 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 235532 T1485 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 235532 T1485 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 235533 T1485 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 235533 T1485 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 235533 T1485 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:41927/solr
   [junit4]   2> 235533 T1485 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 235533 T1485 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 235534 T1485 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 235536 T1508 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5b119511 name:ZooKeeperConnection Watcher:127.0.0.1:41927 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 235536 T1485 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 235537 T1485 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 235538 T1485 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 235540 T1511 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@22cb1792 name:ZooKeeperConnection Watcher:127.0.0.1:41927/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 235540 T1485 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 235541 T1485 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 235543 T1485 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 235545 T1485 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 235547 T1485 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 235549 T1485 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 235551 T1485 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 235552 T1485 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 235553 T1485 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 235554 T1485 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41937_
   [junit4]   2> 235555 T1485 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:41937_
   [junit4]   2> 235556 T1485 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 235557 T1485 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 235558 T1485 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 235560 T1485 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:41937_
   [junit4]   2> 235560 T1485 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 235561 T1485 oasc.Overseer.start Overseer (id=93454583062528003-127.0.0.1:41937_-n_0000000000) starting
   [junit4]   2> 235563 T1485 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 235568 T1485 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 235568 T1513 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 235568 T1485 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 235569 T1512 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 235572 T1485 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/control-001/cores
   [junit4]   1> CORE DESCRIPTOR: {loadOnStartup=true, collection=control_collection, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/control-001/cores/collection1/, dataDir=data/, name=collection1, schema=schema.xml, shard=, coreNodeName=, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/control-001/cores/collection1, config=solrconfig.xml, transient=false}
   [junit4]   2> 235574 T1485 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/control-001/cores/collection1/
   [junit4]   2> 235574 T1485 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 235575 T1515 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 235575 T1515 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 235576 T1511 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 235576 T1515 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 235576 T1512 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:41937_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41937",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"} current state version: 0
   [junit4]   2> 235577 T1512 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:41937_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41937",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 235577 T1512 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 235577 T1512 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 235578 T1511 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 236576 T1515 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 236577 T1515 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 236577 T1515 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 236577 T1515 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 236578 T1515 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 236578 T1515 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/control-001/cores/collection1/'
   [junit4]   2> 236592 T1515 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 236594 T1515 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 236604 T1515 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 236622 T1515 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 236661 T1515 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 236662 T1515 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 236670 T1515 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 237018 T1515 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 237029 T1515 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 237031 T1515 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 237042 T1515 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 237046 T1515 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 237049 T1515 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 237050 T1515 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 237050 T1515 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 237051 T1515 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 237051 T1515 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 237051 T1515 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 237052 T1515 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 237052 T1515 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 237052 T1515 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/control-001/cores/collection1/data/
   [junit4]   2> 237052 T1515 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6b38579e
   [junit4]   2> 237054 T1515 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/control-001/cores/collection1/data
   [junit4]   2> 237055 T1515 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/control-001/cores/collection1/data/index/
   [junit4]   2> 237055 T1515 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 237056 T1515 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/control-001/cores/collection1/data/index
   [junit4]   2> 237056 T1515 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=47, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.3030080595586424]
   [junit4]   2> 237057 T1515 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@690ce05 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@cc5ada1; maxCacheMB=0.7509765625 maxMergeSizeMB=0.8154296875)),segFN=segments_1,generation=1}
   [junit4]   2> 237057 T1515 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 237062 T1515 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 237062 T1515 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 237062 T1515 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 237062 T1515 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 237062 T1515 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 237063 T1515 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 237063 T1515 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 237063 T1515 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 237063 T1515 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 237089 T1515 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 237091 T1515 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 237093 T1515 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 237095 T1515 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 237099 T1515 oasc.RequestHandlers.initHandlersFromConfig Registered paths: /update,/schema,/update/csv,/admin/segments,/admin/properties,/replication,/config,/admin/luke,/admin/plugins,/admin/file,/update/json,/admin/ping,/update/json/docs,/get,/admin/logging,/admin/system,/admin/mbeans,standard,/admin/threads
   [junit4]   2> 237101 T1515 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 237103 T1515 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 237103 T1515 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 237104 T1515 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 237105 T1515 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 237105 T1515 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=42, maxMergeAtOnceExplicit=11, maxMergedSegmentMB=0.54296875, floorSegmentMB=1.7041015625, forceMergeDeletesPctAllowed=19.582671415679087, segmentsPerTier=47.0, maxCFSSegmentSizeMB=0.7294921875, noCFSRatio=0.0
   [junit4]   2> 237106 T1515 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@690ce05 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@cc5ada1; maxCacheMB=0.7509765625 maxMergeSizeMB=0.8154296875)),segFN=segments_1,generation=1}
   [junit4]   2> 237106 T1515 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 237106 T1515 oass.SolrIndexSearcher.<init> Opening Searcher@61795feb[collection1] main
   [junit4]   2> 237107 T1515 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 237107 T1515 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 237108 T1515 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 237108 T1515 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 237108 T1515 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 237108 T1515 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 237109 T1515 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 237109 T1515 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 237109 T1515 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 237109 T1515 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 237110 T1516 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@61795feb[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 237111 T1515 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 237111 T1519 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:41937 collection:control_collection shard:shard1
   [junit4]   2> 237112 T1485 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 237112 T1485 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 237112 T1519 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 237115 T1485 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 237116 T1485 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 237116 T1519 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 237118 T1523 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1e72c853 name:ZooKeeperConnection Watcher:127.0.0.1:41927/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 237118 T1485 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 237118 T1511 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 237119 T1485 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 237119 T1519 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 237119 T1512 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 237119 T1519 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 237119 T1485 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C149 name=collection1 org.apache.solr.core.SolrCore@662a6d5b url=http://127.0.0.1:41937/collection1 node=127.0.0.1:41937_ C149_STATE=coll:control_collection core:collection1 props:{core=collection1, node_name=127.0.0.1:41937_, base_url=http://127.0.0.1:41937, state=down}
   [junit4]   2> 237119 T1519 C149 P41937 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:41937/collection1/
   [junit4]   2> 237121 T1519 C149 P41937 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 237121 T1523 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 237121 T1511 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 237121 T1519 C149 P41937 oasc.SyncStrategy.syncToMe http://127.0.0.1:41937/collection1/ has no replicas
   [junit4]   2> 237122 T1519 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:41937/collection1/ shard1
   [junit4]   2> 237122 T1519 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 237123 T1485 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 237123 T1485 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 237123 T1485 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 237124 T1485 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 237125 T1526 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@764af7ed name:ZooKeeperConnection Watcher:127.0.0.1:41927/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 237125 T1485 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 237126 T1485 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 237126 T1511 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 237127 T1512 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41937",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 237130 T1512 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 237130 T1512 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 237130 T1523 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 237130 T1511 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 237133 T1511 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 237133 T1512 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 237134 T1511 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 237134 T1523 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 237177 T1519 oasc.ZkController.register We are http://127.0.0.1:41937/collection1/ and leader is http://127.0.0.1:41937/collection1/
   [junit4]   2> 237178 T1519 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:41937
   [junit4]   2> 237178 T1519 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 237178 T1519 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 237178 T1519 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 237179 T1511 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 237180 T1512 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:41937_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41937",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state"} current state version: 4
   [junit4]   2> 237181 T1512 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "node_name":"127.0.0.1:41937_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41937",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 237284 T1511 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 237284 T1523 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 238669 T1485 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/shard-1-001/cores/collection1
   [junit4]   2> 238671 T1485 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/shard-1-001
   [junit4]   2> 238672 T1485 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 238674 T1485 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:41959
   [junit4]   2> 238674 T1485 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/shard-1-001/cores, solrconfig=solrconfig.xml, hostContext=/, hostPort=41958}
   [junit4]   2> 238675 T1485 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 238675 T1485 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/shard-1-001/'
   [junit4]   2> 238708 T1485 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/shard-1-001/solr.xml
   [junit4]   2> 238748 T1485 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/shard-1-001/cores
   [junit4]   2> 238748 T1485 oasc.CoreContainer.<init> New CoreContainer 294987646
   [junit4]   2> 238748 T1485 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/shard-1-001/]
   [junit4]   2> 238749 T1485 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/shard-1-001/lib
   [junit4]   2> 238749 T1485 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/shard-1-001/lib).
   [junit4]   2> 238759 T1485 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 238759 T1485 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 238759 T1485 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 238759 T1485 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 238760 T1485 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 238760 T1485 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 238760 T1485 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 238760 T1485 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 238760 T1485 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 238761 T1485 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 238761 T1485 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 238761 T1485 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 238762 T1485 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 238762 T1485 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 238762 T1485 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 238762 T1485 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:41927/solr
   [junit4]   2> 238762 T1485 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 238763 T1485 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 238763 T1485 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 238765 T1538 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4ff21fbe name:ZooKeeperConnection Watcher:127.0.0.1:41927 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 238766 T1485 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 238766 T1485 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 238768 T1485 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 238769 T1541 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@573b98bb name:ZooKeeperConnection Watcher:127.0.0.1:41927/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 238770 T1485 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 238773 T1485 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 239779 T1485 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:41958_
   [junit4]   2> 239780 T1485 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:41958_
   [junit4]   2> 239783 T1485 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 239786 T1485 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/shard-1-001/cores
   [junit4]   1> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/shard-1-001/cores/collection1/, config=solrconfig.xml, coreNodeName=, collection=collection1, shard=, transient=false, dataDir=data/, name=collection1, schema=schema.xml, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/shard-1-001/cores/collection1}
   [junit4]   2> 239789 T1485 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/shard-1-001/cores/collection1/
   [junit4]   2> 239789 T1485 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 239792 T1542 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 239792 T1542 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 239794 T1511 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 239794 T1542 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 239794 T1542 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 239794 T1512 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:41958_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41958",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"} current state version: 5
   [junit4]   2> 239795 T1542 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 239795 T1512 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:41958_",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:41958",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state"}
   [junit4]   2> 239795 T1512 oasco.ReplicaMutator.updateState Collection already exists with numShards=1
   [junit4]   2> 239796 T1512 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 239796 T1542 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 239796 T1542 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 239797 T1512 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 239798 T1541 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 239798 T1541 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 240797 T1542 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 240798 T1542 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 240799 T1542 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 240799 T1542 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 240800 T1542 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 240800 T1542 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/shard-1-001/cores/collection1/'
   [junit4]   2> 240819 T1542 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 240822 T1542 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 240837 T1542 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 240863 T1542 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 240919 T1542 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 240921 T1542 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 240932 T1542 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 241373 T1542 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 241388 T1542 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 241391 T1542 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 241405 T1542 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 241410 T1542 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 241415 T1542 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 241416 T1542 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 241416 T1542 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 241416 T1542 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 241417 T1542 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 241418 T1542 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 241418 T1542 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 241418 T1542 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 241418 T1542 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 241419 T1542 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@6b38579e
   [junit4]   2> 241422 T1542 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/shard-1-001/cores/collection1/data
   [junit4]   2> 241423 T1542 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/shard-1-001/cores/collection1/data/index/
   [junit4]   2> 241423 T1542 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.RecoveryAfterSoftCommitTest 44060DA38D0FC4E9-001/shard-1-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 241424 T1542 oasc.Cac

[...truncated too long message...]

s.txt -Dtests.locale=is_IS -Dtests.timezone=America/Indiana/Marengo -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   67.8s J2 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18962/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([44060DA38D0FC4E9:CC52327923F3A911]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:625)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:948)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:839)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:782)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:18962/repfacttest_c8n_1x3_shard1_replica1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:622)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	... 53 more
   [junit4]   2> 2162110 T6020 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest 44060DA38D0FC4E9-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {intDefault=FSTOrd50, _version_=FSTOrd50, text=PostingsFormat(name=Memory doPackFST= true), a_t=PostingsFormat(name=LuceneVarGapDocFreqInterval), id=PostingsFormat(name=LuceneFixedGap), timestamp=PostingsFormat(name=LuceneVarGapDocFreqInterval), range_facet_l=PostingsFormat(name=LuceneVarGapDocFreqInterval), multiDefault=PostingsFormat(name=LuceneVarGapDocFreqInterval)}, docValues:{timestamp=DocValuesFormat(name=Asserting)}, sim=DefaultSimilarity, locale=is_IS, timezone=America/Indiana/Marengo
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=145275040,total=395837440
   [junit4]   2> NOTE: All tests run in this JVM: [TestBinaryField, TestInitParams, TestLuceneMatchVersion, SynonymTokenizerTest, TestFoldingMultitermQuery, TestCollapseQParserPlugin, AssignTest, TestSerializedLuceneMatchVersion, TestNRTOpen, NoCacheHeaderTest, SpatialFilterTest, SolrCoreCheckLockOnStartupTest, TestRemoteStreaming, TestRTGBase, TestConfigReload, VMParamsZkACLAndCredentialsProvidersTest, BadCopyFieldTest, TestManagedSchemaFieldResource, CurrencyFieldOpenExchangeTest, SaslZkACLProviderTest, ChaosMonkeyNothingIsSafeTest, ReturnFieldsTest, TestExactSharedStatsCache, SolrRequestParserTest, DirectUpdateHandlerTest, TestElisionMultitermQuery, SOLR749Test, TestReload, TestExceedMaxTermLength, SolrIndexSplitterTest, TestReqParamsAPI, PeerSyncTest, TestMergePolicyConfig, TestDistribDocBasedVersion, TestSolrConfigHandler, MigrateRouteKeyTest, DocumentBuilderTest, TestLRUCache, TestMaxScoreQueryParser, StatsComponentTest, TestComponentsName, BufferStoreTest, CachingDirectoryFactoryTest, TestSolr4Spatial2, FileBasedSpellCheckerTest, TestRealTimeGet, CoreAdminRequestStatusTest, TestSolrDynamicMBean, TestExtendedDismaxParser, TestDistributedSearch, TestFastWriter, LeaderFailoverAfterPartitionTest, AnalyticsMergeStrategyTest, RAMDirectoryFactoryTest, TestDocBasedVersionConstraints, SimpleMLTQParserTest, SharedFSAutoReplicaFailoverUtilsTest, SuggestComponentTest, ClusterStateUpdateTest, TestRandomFaceting, TestHashPartitioner, TermVectorComponentDistributedTest, TestStressVersions, SpellCheckComponentTest, SolrCmdDistributorTest, QueryElevationComponentTest, TestBadConfig, OverseerCollectionProcessorTest, DistributedQueryElevationComponentTest, SimplePostToolTest, SuggesterFSTTest, CoreAdminHandlerTest, WordBreakSolrSpellCheckerTest, TestWordDelimiterFilterFactory, DirectUpdateHandlerOptimizeTest, DefaultValueUpdateProcessorTest, SolrInfoMBeanTest, DebugComponentTest, LukeRequestHandlerTest, StandardRequestHandlerTest, TestReversedWildcardFilterFactory, TestQueryTypes, TermsComponentTest, RequestHandlersTest, PathHierarchyTokenizerFactoryTest, MoreLikeThisHandlerTest, TestArbitraryIndexDir, LoggingHandlerTest, SolrPluginUtilsTest, TestJmxIntegration, TestCSVResponseWriter, TestAnalyzedSuggestions, TestPHPSerializedResponseWriter, CSVRequestHandlerTest, TestLFUCache, TestQuerySenderNoQuery, BadComponentTest, TestStressRecovery, TestConfig, DirectSolrConnectionTest, TestPhraseSuggestions, TestCodecSupport, TestPerFieldSimilarity, TestBM25SimilarityFactory, URLClassifyProcessorTest, DateFieldTest, CursorPagingTest, SolrTestCaseJ4Test, TestDistributedMissingSort, TestTolerantSearch, AsyncMigrateRouteKeyTest, DistribCursorPagingTest, LeaderInitiatedRecoveryOnCommitTest, OutOfBoxZkACLAndCredentialsProvidersTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerRolesTest, ReplicationFactorTest]
   [junit4] Completed on J2 in 67.83s, 1 test, 1 error <<< FAILURES!

[...truncated 309 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 472 suites, 1878 tests, 5 errors, 58 ignored (9 assumptions)

Total time: 207 minutes 13 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 721489804 bytes
Compression is 0.0%
Took 2 min 13 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 781 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/781/

5 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([9DD6EFD23DFF2A4:818951278D239F5C]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:927)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:780)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:594)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:591)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([9DD6EFD23DFF2A4:818951278D239F5C]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:927)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:780)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:594)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:591)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:62187/i/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:62187/i/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([9DD6EFD23DFF2A4:818951278D239F5C]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:598)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([9DD6EFD23DFF2A4:818951278D239F5C]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:927)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:780)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:594)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:591)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([9DD6EFD23DFF2A4:818951278D239F5C]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:927)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:780)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:594)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:591)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 10051 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/init-core-data-001
   [junit4]   2> 166400 T415 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /_w/x
   [junit4]   2> 166410 T415 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 166412 T416 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 166512 T415 oasc.ZkTestServer.run start zk server on port:59570
   [junit4]   2> 166514 T415 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 166515 T415 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 166522 T423 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7b8c6ef2 name:ZooKeeperConnection Watcher:127.0.0.1:59570 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 166522 T415 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 166524 T415 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 166524 T415 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 166541 T415 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 166543 T415 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 166546 T426 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@393105bc name:ZooKeeperConnection Watcher:127.0.0.1:59570/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 166547 T415 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 166548 T415 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 166548 T415 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 166553 T415 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 166556 T415 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 166559 T415 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 166563 T415 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 166564 T415 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 166568 T415 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 166569 T415 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 166573 T415 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 166575 T415 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 166578 T415 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 166579 T415 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 166583 T415 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 166584 T415 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 166588 T415 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 166590 T415 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 166594 T415 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 166595 T415 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 166598 T415 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 166599 T415 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 166603 T415 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 166604 T415 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 166608 T415 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 166608 T415 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 166611 T415 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 166612 T415 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 169418 T415 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/control-001/cores/collection1
   [junit4]   2> 169423 T415 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 169463 T415 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:59603
   [junit4]   2> 169477 T415 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/control-001/cores, hostContext=/_w/x, hostPort=59602, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/tempDir-001/control/data}
   [junit4]   2> 169500 T415 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 169500 T415 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/control-001/'
   [junit4]   2> 169680 T415 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/control-001/solr.xml
   [junit4]   2> 169857 T415 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/control-001/cores
   [junit4]   2> 169858 T415 oasc.CoreContainer.<init> New CoreContainer 1832180056
   [junit4]   2> 169858 T415 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/control-001/]
   [junit4]   2> 169859 T415 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/control-001/lib
   [junit4]   2> 169859 T415 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/control-001/lib).
   [junit4]   2> 169934 T415 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 169935 T415 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 169935 T415 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 169935 T415 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 169936 T415 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 169936 T415 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 169937 T415 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 169937 T415 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 169937 T415 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 169938 T415 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 169938 T415 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 169939 T415 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 169940 T415 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 169944 T415 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 169956 T415 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 169958 T415 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:59570/solr
   [junit4]   2> 169968 T415 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 169975 T415 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 169983 T415 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 170000 T438 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@17587455 name:ZooKeeperConnection Watcher:127.0.0.1:59570 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 170001 T415 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 170002 T415 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 170005 T415 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 170007 T441 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@8c78a0 name:ZooKeeperConnection Watcher:127.0.0.1:59570/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 170007 T415 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 170048 T415 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 170050 T415 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 170069 T415 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 170108 T415 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 170118 T415 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 170125 T415 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 170135 T415 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 170137 T415 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 170171 T415 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59602__w%2Fx
   [junit4]   2> 170193 T415 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:59602__w%2Fx
   [junit4]   2> 170200 T415 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 170208 T415 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 170210 T415 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 170230 T415 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:59602__w%2Fx
   [junit4]   2> 170233 T415 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 170248 T415 oasc.Overseer.start Overseer (id=93448897499430915-127.0.0.1:59602__w%2Fx-n_0000000000) starting
   [junit4]   2> 170273 T415 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 170321 T415 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 170324 T443 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 170324 T415 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 170331 T442 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 170432 T415 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/control-001/cores
   [junit4]   1> CORE DESCRIPTOR: {config=solrconfig.xml, loadOnStartup=true, transient=false, shard=, dataDir=data/, collection=control_collection, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/control-001/cores/collection1, name=collection1, coreNodeName=, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/control-001/cores/collection1/}
   [junit4]   2> 170443 T415 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/control-001/cores/collection1/
   [junit4]   2> 170444 T415 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 170495 T445 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 170495 T445 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 170509 T441 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 170509 T445 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 170510 T442 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:59602__w%2Fx",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59602/_w/x"} current state version: 0
   [junit4]   2> 170515 T442 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:59602__w%2Fx",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59602/_w/x"}
   [junit4]   2> 170519 T442 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 170522 T442 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 170549 T441 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 171511 T445 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 171511 T445 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 171515 T445 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 171516 T445 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 171520 T445 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 171520 T445 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/control-001/cores/collection1/'
   [junit4]   2> 171549 T445 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 171566 T445 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 171606 T445 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 171677 T445 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 171728 T445 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 171730 T445 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 171781 T445 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 173547 T445 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 173629 T445 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 173634 T445 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 173704 T445 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 173767 T445 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 173775 T445 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 173778 T445 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 173779 T445 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 173779 T445 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 173780 T445 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 173780 T445 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 173780 T445 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 173781 T445 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 173781 T445 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/control-001/cores/collection1/data/
   [junit4]   2> 173783 T445 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@1bd26bee
   [junit4]   2> 173786 T445 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/control-001/cores/collection1/data
   [junit4]   2> 173786 T445 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/control-001/cores/collection1/data/index/
   [junit4]   2> 173787 T445 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 173787 T445 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/control-001/cores/collection1/data/index
   [junit4]   2> 173793 T445 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1829564849, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 173815 T445 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@482c32c6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4db98a2e; maxCacheMB=0.314453125 maxMergeSizeMB=0.0986328125)),segFN=segments_1,generation=1}
   [junit4]   2> 173818 T445 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 173843 T445 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 173845 T445 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 173846 T445 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 173846 T445 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 173846 T445 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 173847 T445 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 173847 T445 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 173848 T445 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 173848 T445 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 173849 T445 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 173850 T445 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 173850 T445 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 173850 T445 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 173851 T445 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 173852 T445 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 173855 T445 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 173856 T445 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 173856 T445 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 173862 T445 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 173871 T445 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 173875 T445 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 173877 T445 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 173886 T445 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 173888 T445 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 173897 T445 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 173910 T445 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 173911 T445 oasc.RequestHandlers.initHandlersFromConfig created /admin/segments: org.apache.solr.handler.admin.SegmentsInfoRequestHandler
   [junit4]   2> 173911 T445 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 174017 T445 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 174019 T445 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 174035 T445 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 174059 T445 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 174080 T445 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 174090 T445 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 174090 T445 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 174099 T445 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 174107 T445 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 174108 T445 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=46, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 174129 T445 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@482c32c6 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@4db98a2e; maxCacheMB=0.314453125 maxMergeSizeMB=0.0986328125)),segFN=segments_1,generation=1}
   [junit4]   2> 174133 T445 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 174149 T445 oass.SolrIndexSearcher.<init> Opening Searcher@7b9efe5f[collection1] main
   [junit4]   2> 174150 T445 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 174151 T445 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 174151 T445 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 174152 T445 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 174153 T445 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 174153 T445 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 174155 T445 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 174157 T445 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 174157 T445 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 174158 T445 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 174159 T446 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7b9efe5f[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 174159 T445 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 174160 T449 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:59602/_w/x collection:control_collection shard:shard1
   [junit4]   2> 174162 T415 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 174162 T449 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 174162 T415 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 174166 T415 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 174168 T415 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 174172 T453 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@21470daa name:ZooKeeperConnection Watcher:127.0.0.1:59570/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 174172 T415 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 174174 T415 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 174174 T415 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 174185 T449 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 174192 T415 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 174193 T415 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 174195 T415 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 174196 T415 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 174204 T456 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@63ee1d8e name:ZooKeeperConnection Watcher:127.0.0.1:59570/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 174204 T415 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 174206 T415 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 174212 T449 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 174212 T441 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 174220 T449 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C37 name=collection1 org.apache.solr.core.SolrCore@2380c601 url=http://127.0.0.1:59602/_w/x/collection1 node=127.0.0.1:59602__w%2Fx C37_STATE=coll:control_collection core:collection1 props:{core=collection1, state=down, node_name=127.0.0.1:59602__w%2Fx, base_url=http://127.0.0.1:59602/_w/x}
   [junit4]   2> 174220 T449 C37 P59602 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:59602/_w/x/collection1/
   [junit4]   2> 174221 T449 C37 P59602 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 174221 T442 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 174221 T449 C37 P59602 oasc.SyncStrategy.syncToMe http://127.0.0.1:59602/_w/x/collection1/ has no replicas
   [junit4]   2> 174221 T449 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:59602/_w/x/collection1/ shard1
   [junit4]   2> 174222 T449 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 174224 T417 oazs.NIOServerCnxn.doIO WARN caught end of stream exception EndOfStreamException: Unable to read additional data from client sessionid 0x14bff46f41b0005, likely client has closed socket
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:228)
   [junit4]   2> 	at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:208)
   [junit4]   2> 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 
   [junit4]   2> 174224 T453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 174224 T441 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 174237 T442 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 2, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 174237 T442 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 174304 T441 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 174308 T442 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 2, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59602/_w/x",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 174311 T442 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 174330 T453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 174331 T441 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 174446 T441 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 174447 T453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 174459 T449 oasc.ZkController.register We are http://127.0.0.1:59602/_w/x/collection1/ and leader is http://127.0.0.1:59602/_w/x/collection1/
   [junit4]   2> 174466 T449 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:59602/_w/x
   [junit4]   2> 174469 T449 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 174469 T449 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 174469 T449 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 174477 T441 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 174482 T442 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:59602__w%2Fx",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59602/_w/x"} current state version: 4
   [junit4]   2> 174483 T442 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:59602__w%2Fx",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59602/_w/x"}
   [junit4]   2> 174600 T441 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 174604 T453 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 178637 T415 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/shard-1-001/cores/collection1
   [junit4]   2> 178640 T415 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/shard-1-001
   [junit4]   2> 178641 T415 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 178643 T415 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:59623
   [junit4]   2> 178644 T415 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/_w/x, hostPort=59622, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/tempDir-001/jetty1, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/shard-1-001/cores, solrconfig=solrconfig.xml}
   [junit4]   2> 178646 T415 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@17072b90
   [junit4]   2> 178646 T415 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/shard-1-001/'
   [junit4]   2> 179177 T415 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/shard-1-001/solr.xml
   [junit4]   2> 179690 T415 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/shard-1-001/cores
   [junit4]   2> 179691 T415 oasc.CoreContainer.<init> New CoreContainer 536625236
   [junit4]   2> 179691 T415 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/shard-1-001/]
   [junit4]   2> 179692 T415 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/shard-1-001/lib
   [junit4]   2> 179693 T415 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/shard-1-001/lib).
   [junit4]   2> 179787 T415 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 179787 T415 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 179787 T415 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 179788 T415 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 179788 T415 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 179788 T415 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 179789 T415 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 179789 T415 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 179789 T415 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 179789 T415 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 179790 T415 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 179795 T415 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 179799 T415 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 179802 T415 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 179821 T415 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 179836 T415 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:59570/solr
   [junit4]   2> 179838 T415 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 179840 T415 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 179851 T415 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 179854 T468 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@77bebe00 name:ZooKeeperConnection Watcher:127.0.0.1:59570 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 179856 T415 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 179857 T415 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 179860 T415 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 179862 T471 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2be45207 name:ZooKeeperConnection Watcher:127.0.0.1:59570/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 179863 T415 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 179921 T415 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 180970 T415 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59622__w%2Fx
   [junit4]   2> 180971 T415 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:59622__w%2Fx
   [junit4]   2> 180978 T415 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 180989 T415 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/shard-1-001/cores
   [junit4]   1> CORE DESCRIPTOR: {dataDir=data/, schema=schema.xml, transient=false, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/shard-1-001/cores/collection1/, shard=, name=collection1, collection=collection1, coreNodeName=, loadOnStartup=true, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/shard-1-001/cores/collection1}
   [junit4]   2> 181047 T415 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/shard-1-001/cores/collection1/
   [junit4]   2> 181048 T415 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 181053 T472 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 181053 T472 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 181055 T441 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 181056 T472 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 181056 T472 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 181056 T442 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:59622__w%2Fx",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59622/_w/x"} current state version: 5
   [junit4]   2> 181056 T472 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 181057 T442 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:59622__w%2Fx",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59622/_w/x"}
   [junit4]   2> 181057 T442 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 181057 T442 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard3
   [junit4]   2> 181057 T472 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 0 
   [junit4]   2> 181058 T472 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 181061 T442 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 181062 T471 oascc.ZkStateReader$7.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/collections/collection1/state.json for collection collection1 has occurred - updating... (live nodes size: 2)
   [junit4]   2> 181064 T471 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 182064 T472 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 182065 T472 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 182089 T472 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 182093 T472 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 182108 T472 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 182108 T472 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/shard-1-001/cores/collection1/'
   [junit4]   2> 182151 T472 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 182178 T472 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 182383 T472 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 182432 T472 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 182677 T472 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 182681 T472 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 182704 T472 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 183256 T472 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 183270 T472 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 183273 T472 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 183324 T472 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 183331 T472 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 183338 T472 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 183339 T472 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 183339 T472 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 183340 T472 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 183340 T472 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 183341 T472 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 183341 T472 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 183341 T472 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 183342 T472 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 9DD6EFD23DFF2A4-001/shard-1-001/cores/collection1/data/
   [junit4

[...truncated too long message...]

ethod=test -Dtests.seed=9DD6EFD23DFF2A4 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=zh_HK -Dtests.timezone=Asia/Kamchatka -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   97.3s J1 | LeaderFailoverAfterPartitionTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([9DD6EFD23DFF2A4:818951278D239F5C]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:927)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:780)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
   [junit4]    > 	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
   [junit4]    > 	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
   [junit4]    > 	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:490)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:236)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:228)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:594)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:591)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 2251651 T7718 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.LeaderFailoverAfterPartitionTest 9DD6EFD23DFF2A4-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {intDefault=PostingsFormat(name=Memory doPackFST= true), id=PostingsFormat(name=LuceneVarGapDocFreqInterval), timestamp=PostingsFormat(name=Memory doPackFST= false), _version_=PostingsFormat(name=Memory doPackFST= true), range_facet_l=PostingsFormat(name=Memory doPackFST= false), a_t=PostingsFormat(name=Memory doPackFST= false), text=PostingsFormat(name=Asserting), multiDefault=PostingsFormat(name=Memory doPackFST= false)}, docValues:{timestamp=DocValuesFormat(name=Asserting)}, sim=RandomSimilarityProvider(queryNorm=false,coord=no): {}, locale=zh_HK, timezone=Asia/Kamchatka
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=60901272,total=409993216
   [junit4]   2> NOTE: All tests run in this JVM: [AutoCommitTest, FullSolrCloudDistribCmdsTest, OverseerRolesTest, TestStressLucene, DocumentAnalysisRequestHandlerTest, PingRequestHandlerTest, TestFunctionQuery, UniqFieldsUpdateProcessorFactoryTest, TestDocumentBuilder, TestReplicaProperties, QueryElevationComponentTest, DistributedQueryComponentCustomSortTest, TestBinaryResponseWriter, TestLFUCache, TestSolrConfigHandlerConcurrent, MultiThreadedOCPTest, TestFuzzyAnalyzedSuggestions, ZkControllerTest, TestRecovery, TestSolr4Spatial2, PathHierarchyTokenizerFactoryTest, TestSimpleTrackingShardHandler, SuggesterTSTTest, TestSolrDeletionPolicy1, DistributedQueryComponentOptimizationTest, TestOrdValues, SharedFSAutoReplicaFailoverTest, JSONWriterTest, DistributedFacetPivotSmallTest, TestSolrConfigHandlerCloud, TestConfigReload, TestConfig, AssignTest, BlockCacheTest, TestOmitPositions, SchemaVersionSpecificBehaviorTest, CopyFieldTest, CircularListTest, TestSerializedLuceneMatchVersion, TestDynamicFieldCollectionResource, TestBulkSchemaConcurrent, ResourceLoaderTest, TermVectorComponentTest, SolrInfoMBeanTest, TestZkChroot, PluginInfoTest, SystemInfoHandlerTest, TestCloudManagedSchema, TestDocBasedVersionConstraints, BasicZkTest, TestStressVersions, TestCopyFieldCollectionResource, TestMaxScoreQueryParser, HighlighterTest, HdfsWriteToMultipleCollectionsTest, BasicDistributedZkTest, LeaderElectionIntegrationTest, RecoveryZkTest, TestRandomFaceting, LeaderElectionTest, ZkSolrClientTest, TestRandomDVFaceting, ZkCLITest, ShardRoutingCustomTest, TestDistributedSearch, SimpleFacetsTest, TestSolr4Spatial, SolrCoreTest, ConvertedLegacyTest, TestFiltering, TestIndexSearcher, SimplePostToolTest, TestCoreDiscovery, SpatialFilterTest, PolyFieldTest, FieldMutatingUpdateProcessorTest, DirectUpdateHandlerOptimizeTest, StatelessScriptUpdateProcessorFactoryTest, SortByFunctionTest, XsltUpdateRequestHandlerTest, TestSurroundQueryParser, StandardRequestHandlerTest, PrimitiveFieldTypeTest, DocumentBuilderTest, TestSolrQueryParser, SolrPluginUtilsTest, UpdateRequestProcessorFactoryTest, QueryParsingTest, TestPartialUpdateDeduplication, SOLR749Test, SolrIndexConfigTest, MinimalSchemaTest, DirectSolrConnectionTest, TestPhraseSuggestions, TestCodecSupport, TestXIncludeConfig, TestSweetSpotSimilarityFactory, TestLMJelinekMercerSimilarityFactory, TestFastWriter, TestJmxMonitoredMap, TestFastOutputStream, DateMathParserTest, TestSystemIdResolver, PrimUtilsTest, TestSuggestSpellingConverter, RAMDirectoryFactoryTest, TestUtils, CursorPagingTest, ConnectionManagerTest, CustomCollectionTest, DistribDocExpirationUpdateProcessorTest, LeaderFailoverAfterPartitionTest]
   [junit4] Completed on J1 in 97.35s, 1 test, 1 error <<< FAILURES!

[...truncated 461 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 471 suites, 1875 tests, 5 errors, 58 ignored (9 assumptions)

Total time: 202 minutes 22 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 753301279 bytes
Compression is 0.0%
Took 2 min 17 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 780 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/780/

6 tests failed.
REGRESSION:  org.apache.solr.cloud.RecoveryZkTest.test

Error Message:
shard1 is not consistent.  Got 114 from http://127.0.0.1:38653/collection1lastClient and got 56 from http://127.0.0.1:38660/collection1

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 114 from http://127.0.0.1:38653/collection1lastClient and got 56 from http://127.0.0.1:38660/collection1
	at __randomizedtesting.SeedInfo.seed([A9DF7A9A0B43324F:218B4540A5BF5FB7]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.RecoveryZkTest.test(RecoveryZkTest.java:123)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([A9DF7A9A0B43324F:218B4540A5BF5FB7]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:901)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:754)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:461)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:211)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:207)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:568)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:565)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([A9DF7A9A0B43324F:218B4540A5BF5FB7]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:901)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:754)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:461)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:211)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:207)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:568)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:565)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:39990/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:39990/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([A9DF7A9A0B43324F:218B4540A5BF5FB7]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:569)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:211)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:207)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:461)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([A9DF7A9A0B43324F:218B4540A5BF5FB7]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:901)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:754)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:461)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:211)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:207)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:568)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:565)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([A9DF7A9A0B43324F:218B4540A5BF5FB7]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:901)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:754)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:461)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:211)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:207)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:568)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:565)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 9979 lines...]
   [junit4] Suite: org.apache.solr.cloud.HttpPartitionTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/init-core-data-001
   [junit4]   2> 51264 T117 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 51330 T117 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 51343 T118 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 51644 T117 oasc.ZkTestServer.run start zk server on port:29035
   [junit4]   2> 51806 T117 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 52072 T117 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 52259 T125 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@af23da5 name:ZooKeeperConnection Watcher:127.0.0.1:29035 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 52259 T117 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 52260 T117 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 52273 T117 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 52411 T117 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 52412 T117 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 52414 T128 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@661f77f1 name:ZooKeeperConnection Watcher:127.0.0.1:29035/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 52414 T117 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 52415 T117 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 52422 T117 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 52426 T117 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 52429 T117 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 52431 T117 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 52434 T117 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 52435 T117 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 52439 T117 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 52440 T117 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 52443 T117 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 52444 T117 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 52447 T117 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 52447 T117 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 52451 T117 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 52451 T117 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 52454 T117 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 52455 T117 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 52458 T117 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 52459 T117 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 52462 T117 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 52462 T117 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 52465 T117 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 52466 T117 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 52469 T117 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 52469 T117 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 52472 T117 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 52473 T117 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 54373 T117 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/control-001/cores/collection1
   [junit4]   2> 55215 T117 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 55664 T117 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:59209
   [junit4]   2> 55665 T117 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/tempDir-001/control/data, hostPort=25751}
   [junit4]   2> 55805 T117 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@749cd006
   [junit4]   2> 55805 T117 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/control-001/'
   [junit4]   2> 55837 T117 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/control-001/solr.xml
   [junit4]   2> 55880 T117 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/control-001/cores
   [junit4]   2> 55880 T117 oasc.CoreContainer.<init> New CoreContainer 997623076
   [junit4]   2> 55880 T117 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/control-001/]
   [junit4]   2> 55881 T117 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/control-001/lib
   [junit4]   2> 55881 T117 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/control-001/lib).
   [junit4]   2> 55948 T117 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 55948 T117 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 55948 T117 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 55949 T117 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 55949 T117 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 55949 T117 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 55949 T117 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 55950 T117 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 55950 T117 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 55950 T117 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 55950 T117 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 55951 T117 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 55951 T117 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 55952 T117 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 55952 T117 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 55952 T117 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:29035/solr
   [junit4]   2> 55985 T117 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 55985 T117 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 55986 T117 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 55989 T141 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4316ebe name:ZooKeeperConnection Watcher:127.0.0.1:29035 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 55989 T117 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 55990 T117 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 56009 T117 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 56011 T144 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@db53b7d name:ZooKeeperConnection Watcher:127.0.0.1:29035/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 56011 T117 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 56029 T117 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 56044 T117 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 56052 T117 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 56056 T117 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 56059 T117 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 56083 T117 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 56085 T117 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:25751_
   [junit4]   2> 56091 T117 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:25751_
   [junit4]   2> 56095 T117 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 56097 T117 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 56100 T117 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 56117 T117 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:25751_
   [junit4]   2> 56117 T117 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 56120 T117 oasc.Overseer.start Overseer (id=93445298015895555-127.0.0.1:25751_-n_0000000000) starting
   [junit4]   2> 56137 T117 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 56170 T117 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 56187 T146 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 56187 T117 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 56190 T117 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 56192 T117 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 56216 T145 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 56222 T117 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/control-001/cores
   [junit4]   1> CORE DESCRIPTOR: {absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/control-001/cores/collection1/, loadOnStartup=true, config=solrconfig.xml, shard=, transient=false, dataDir=data/, collection=control_collection, schema=schema.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/control-001/cores/collection1, name=collection1, coreNodeName=}
   [junit4]   2> 56238 T117 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/control-001/cores/collection1/
   [junit4]   2> 56239 T117 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 56240 T148 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 56240 T148 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 56256 T144 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 56256 T148 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 56263 T145 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25751",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:25751_"} current state version: 0
   [junit4]   2> 56318 T145 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25751",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:25751_"}
   [junit4]   2> 56325 T145 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 56372 T145 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 56389 T144 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 57258 T148 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 57258 T148 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 57259 T148 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 57259 T148 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 57261 T148 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 57261 T148 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/control-001/cores/collection1/'
   [junit4]   2> 57310 T148 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 57332 T148 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 57371 T148 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 57404 T148 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 57485 T148 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 57487 T148 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 57569 T148 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 58139 T148 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 58159 T148 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 58164 T148 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 58255 T148 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 58262 T148 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 58270 T148 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 58273 T148 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 58273 T148 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 58273 T148 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 58275 T148 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 58276 T148 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 58277 T148 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 58277 T148 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 58278 T148 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/control-001/cores/collection1/data/
   [junit4]   2> 58279 T148 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@66cee8b1
   [junit4]   2> 58301 T148 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/control-001/cores/collection1/data
   [junit4]   2> 58301 T148 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/control-001/cores/collection1/data/index/
   [junit4]   2> 58302 T148 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 58302 T148 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/control-001/cores/collection1/data/index
   [junit4]   2> 58303 T148 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=35, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=false, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.4803691392801258]
   [junit4]   2> 58304 T148 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5cba372f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@457d1e38),segFN=segments_1,generation=1}
   [junit4]   2> 58304 T148 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 58315 T148 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 58315 T148 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 58316 T148 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 58316 T148 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 58316 T148 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 58317 T148 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 58318 T148 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 58318 T148 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 58318 T148 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 58330 T148 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 58331 T148 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 58332 T148 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 58333 T148 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 58334 T148 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 58336 T148 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 58337 T148 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 58338 T148 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 58339 T148 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 58341 T148 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 58342 T148 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 58344 T148 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 58345 T148 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 58346 T148 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 58348 T148 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 58349 T148 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 58350 T148 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 58351 T148 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 58373 T148 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 58376 T148 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 58378 T148 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 58381 T148 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 58385 T148 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 58387 T148 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 58387 T148 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 58389 T148 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 58390 T148 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 58392 T148 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=11, maxMergeAtOnceExplicit=21, maxMergedSegmentMB=18.810546875, floorSegmentMB=1.0966796875, forceMergeDeletesPctAllowed=22.307832850528797, segmentsPerTier=42.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.6696094113066445
   [junit4]   2> 58392 T148 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@5cba372f lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@457d1e38),segFN=segments_1,generation=1}
   [junit4]   2> 58393 T148 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 58393 T148 oass.SolrIndexSearcher.<init> Opening Searcher@406f0b98[collection1] main
   [junit4]   2> 58394 T148 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 58396 T148 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 58396 T148 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 58415 T148 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 58415 T148 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 58415 T148 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 58417 T148 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 58417 T148 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 58417 T148 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 58418 T148 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 58421 T149 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@406f0b98[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 58429 T148 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 58430 T152 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:25751 collection:control_collection shard:shard1
   [junit4]   2> 58431 T117 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 58432 T117 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 58477 T152 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 58489 T152 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 58493 T144 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 58494 T152 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 58494 T152 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 58495 T145 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> ASYNC  NEW_CORE C51 name=collection1 org.apache.solr.core.SolrCore@70409831 url=http://127.0.0.1:25751/collection1 node=127.0.0.1:25751_ C51_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=http://127.0.0.1:25751, state=down, node_name=127.0.0.1:25751_}
   [junit4]   2> 58505 T152 C51 P25751 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:25751/collection1/
   [junit4]   2> 58506 T152 C51 P25751 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 58507 T152 C51 P25751 oasc.SyncStrategy.syncToMe http://127.0.0.1:25751/collection1/ has no replicas
   [junit4]   2> 58507 T152 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:25751/collection1/ shard1
   [junit4]   2> 58509 T144 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 58525 T152 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 58534 T144 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 58536 T145 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25751",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 58573 T117 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 58575 T117 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 58577 T156 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3344ec94 name:ZooKeeperConnection Watcher:127.0.0.1:29035/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 58577 T117 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 58578 T117 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 58580 T117 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 58591 T117 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 58591 T117 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 58592 T117 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 58592 T117 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 58595 T159 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@38254819 name:ZooKeeperConnection Watcher:127.0.0.1:29035/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 58595 T117 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 58596 T117 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 58600 T144 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 58602 T145 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 58603 T145 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 58604 T144 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 58604 T156 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 58638 T152 oasc.ZkController.register We are http://127.0.0.1:25751/collection1/ and leader is http://127.0.0.1:25751/collection1/
   [junit4]   2> 58638 T152 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:25751
   [junit4]   2> 58638 T152 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 58638 T152 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 58639 T152 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 58641 T144 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 58643 T145 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25751",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:25751_"} current state version: 3
   [junit4]   2> 58645 T145 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:25751",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:25751_"}
   [junit4]   2> 58646 T145 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 58649 T144 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 58649 T156 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 58760 T144 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 58760 T156 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 60483 T117 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/shard-1-001/cores/collection1
   [junit4]   2> 60488 T117 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/shard-1-001
   [junit4]   2> 60489 T117 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 60492 T117 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:61367
   [junit4]   2> 60492 T117 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {hostContext=/, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/tempDir-001/jetty1, solrconfig=solrconfig.xml, hostPort=49490, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/shard-1-001/cores}
   [junit4]   2> 60493 T117 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@749cd006
   [junit4]   2> 60493 T117 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/shard-1-001/'
   [junit4]   2> 60556 T117 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/shard-1-001/solr.xml
   [junit4]   2> 60625 T117 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/shard-1-001/cores
   [junit4]   2> 60626 T117 oasc.CoreContainer.<init> New CoreContainer 1789617437
   [junit4]   2> 60626 T117 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/shard-1-001/]
   [junit4]   2> 60626 T117 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/shard-1-001/lib
   [junit4]   2> 60628 T117 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/shard-1-001/lib).
   [junit4]   2> 60642 T117 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 60642 T117 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 60642 T117 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 60643 T117 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 60643 T117 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 60643 T117 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 60644 T117 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 60644 T117 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 60644 T117 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 60645 T117 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 60645 T117 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 60646 T117 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 60646 T117 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 60646 T117 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 60647 T117 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 60647 T117 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:29035/solr
   [junit4]   2> 60647 T117 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 60648 T117 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 60649 T117 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 60651 T171 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@463a7db0 name:ZooKeeperConnection Watcher:127.0.0.1:29035 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 60652 T117 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 60653 T117 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 60656 T117 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 60659 T174 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3d966f8a name:ZooKeeperConnection Watcher:127.0.0.1:29035/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 60659 T117 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 60670 T117 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 61680 T117 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:49490_
   [junit4]   2> 61682 T117 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:49490_
   [junit4]   2> 61687 T117 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 61696 T117 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/shard-1-001/cores
   [junit4]   1> CORE DESCRIPTOR: {dataDir=data/, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/shard-1-001/cores/collection1, coreNodeName=, name=collection1, transient=false, schema=schema.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/shard-1-001/cores/collection1/, shard=, collection=collection1, config=solrconfig.xml, loadOnStartup=true}
   [junit4]   2> 61698 T117 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest A9DF7A9A0B43324F-001/shard-1-001/cores/collection1/
   [junit4]   2> 61698 T117 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 61703 T175 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 61703 T175 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 61706 T144 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 61708 T175 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 61708 T145 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49490",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:49490_"} current state version: 5
   [junit4]   2> 61708 T175 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 61709 T145 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:49490",
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:49490_"}
   [junit4]   2> 61710 T175 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 61710 T145 oasco.ReplicaMutator.updateState Collection already exists with numShards=2
   [junit4]   2> 61710 T145 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 61713 T145 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 61717 T175 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 61717 T175 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 61717 T175 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 61718 T175 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 61718 T175 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 61718 T175 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 61720 T175 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 61720 T175 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr

[...truncated too long message...]

ped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 1834183 T2148 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:18375 18375
   [junit4]   2> 1834236 T2149 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:18375 18375
   [junit4]   2> 1834237 T2149 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		5	/solr/aliases.json
   [junit4]   2> 		5	/solr/clusterstate.json
   [junit4]   2> 		4	/solr/configs/conf1
   [junit4]   2> 		3	/solr/collections/c8n_1x3_lf/state.json
   [junit4]   2> 		3	/solr/collections/collection1/state.json
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		2	/solr/collections/c8n_1x3_lf/leader_elect/shard1/election/93445408732676107-core_node1-n_0000000000
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		5	/solr/live_nodes
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 		4	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 1834237 T2148 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:18387/
   [junit4]   2> 1834238 T2148 oasc.SocketProxy.close WARN Closing 1 connections to: http://127.0.0.1:18378/
   [junit4]   2> 1834238 T2148 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:18398/
   [junit4]   2> 1834238 T2148 oasc.SocketProxy.close WARN Closing 11 connections to: http://127.0.0.1:18409/
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderFailoverAfterPartitionTest -Dtests.method=test -Dtests.seed=A9DF7A9A0B43324F -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=da_DK -Dtests.timezone=Europe/Vatican -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR   93.1s J2 | LeaderFailoverAfterPartitionTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([A9DF7A9A0B43324F:218B4540A5BF5FB7]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:901)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:754)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
   [junit4]    > 	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
   [junit4]    > 	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
   [junit4]    > 	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:461)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:211)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:207)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:568)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:565)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 1834249 T2148 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest A9DF7A9A0B43324F-001
   [junit4]   2> 93119 T2147 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {timestamp=Lucene50(blocksize=128), text=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), range_facet_l=Lucene50(blocksize=128), intDefault=Lucene50(blocksize=128), multiDefault=FST50, a_t=Lucene50(blocksize=128), _version_=Lucene50(blocksize=128), id=FSTOrd50}, docValues:{timestamp=DocValuesFormat(name=Memory)}, sim=RandomSimilarityProvider(queryNorm=true,coord=crazy): {}, locale=da_DK, timezone=Europe/Vatican
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=101366992,total=277348352
   [junit4]   2> NOTE: All tests run in this JVM: [SOLR749Test, TestReplicationHandler, TestNoOpRegenerator, DirectUpdateHandlerOptimizeTest, HdfsRecoveryZkTest, TestBM25SimilarityFactory, DirectUpdateHandlerTest, TestRandomDVFaceting, TestReRankQParserPlugin, TestElisionMultitermQuery, SimpleMLTQParserTest, TestHdfsUpdateLog, SyncSliceTest, TestReloadAndDeleteDocs, TestReqParamsAPI, FileBasedSpellCheckerTest, TestSchemaVersionResource, TestQuerySenderNoQuery, VMParamsZkACLAndCredentialsProvidersTest, TestBinaryField, ChaosMonkeyNothingIsSafeTest, TestFreeTextSuggestions, TestHashPartitioner, LeaderElectionIntegrationTest, AddSchemaFieldsUpdateProcessorFactoryTest, TestClassNameShortening, CachingDirectoryFactoryTest, TestPHPSerializedResponseWriter, StatsComponentTest, LeaderFailoverAfterPartitionTest]
   [junit4] Completed on J2 in 93.40s, 1 test, 1 error <<< FAILURES!

[...truncated 386 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 470 suites, 1873 tests, 5 errors, 1 failure, 58 ignored (9 assumptions)

Total time: 180 minutes 31 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 696657554 bytes
Compression is 0.0%
Took 2 min 5 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 779 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/779/

5 tests failed.
FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:30213/_uuy/z/c8n_1x2_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:30213/_uuy/z/c8n_1x2_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([C4C1E7A681E74225:4C95D87C2F1B2FDD]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:597)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:920)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:811)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:754)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:30213/_uuy/z/c8n_1x2_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:569)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:211)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:207)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:594)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:461)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:47947/c8n_1x3_lf_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:47947/c8n_1x3_lf_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([C4C1E7A681E74225:4C95D87C2F1B2FDD]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:597)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:920)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:811)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:754)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:47947/c8n_1x3_lf_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:569)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:211)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:207)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:594)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:461)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:17408/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:17408/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([C4C1E7A681E74225:4C95D87C2F1B2FDD]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:569)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:211)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:207)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:461)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38051/b/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38051/b/collection1
	at __randomizedtesting.SeedInfo.seed([C4C1E7A681E74225:4C95D87C2F1B2FDD]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:597)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:920)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:811)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:754)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38051/b/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:569)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:211)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:207)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:594)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:461)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50240/tp_hyu/b/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50240/tp_hyu/b/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([C4C1E7A681E74225:4C95D87C2F1B2FDD]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:597)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:920)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:811)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:754)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:50240/tp_hyu/b/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:569)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:211)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:207)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:594)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:461)
	... 53 more




Build Log:
[...truncated 9875 lines...]
   [junit4] Suite: org.apache.solr.cloud.ReplicationFactorTest
   [junit4]   2> log4j:WARN No such property [conversionPattern] in org.apache.solr.util.SolrLogLayout.
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/init-core-data-001
   [junit4]   2> 4664 T11 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /tp_hyu/b
   [junit4]   2> 4912 T11 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 4928 T12 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 5329 T11 oasc.ZkTestServer.run start zk server on port:36536
   [junit4]   2> 5581 T11 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 5944 T11 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 6238 T19 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@523ed23c name:ZooKeeperConnection Watcher:127.0.0.1:36536 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 6239 T11 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 6257 T11 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 6274 T11 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 6452 T11 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 6454 T11 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 6457 T22 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7118b81f name:ZooKeeperConnection Watcher:127.0.0.1:36536/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 6459 T11 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 6461 T11 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 6497 T11 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 6506 T11 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 6511 T11 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 6516 T11 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 6523 T11 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 6612 T11 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 6620 T11 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 6623 T11 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 6628 T11 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 6630 T11 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 6636 T11 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 6638 T11 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 6643 T11 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 6645 T11 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 6650 T11 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 6652 T11 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 6658 T11 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 6660 T11 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 6665 T11 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 6667 T11 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 6672 T11 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 6674 T11 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 6679 T11 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 6681 T11 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 6686 T11 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 6688 T11 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 9225 T11 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/control-001/cores/collection1
   [junit4]   2> 10403 T11 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 10984 T11 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:19652
   [junit4]   2> 10985 T11 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/control-001/cores, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/tempDir-001/control/data, hostContext=/tp_hyu/b, hostPort=59184}
   [junit4]   2> 11521 T11 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 11657 T11 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/control-001/'
   [junit4]   2> 12829 T11 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/control-001/solr.xml
   [junit4]   2> 13106 T11 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/control-001/cores
   [junit4]   2> 13151 T11 oasc.CoreContainer.<init> New CoreContainer 499510921
   [junit4]   2> 13152 T11 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/control-001/]
   [junit4]   2> 13153 T11 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/control-001/lib
   [junit4]   2> 13154 T11 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/control-001/lib).
   [junit4]   2> 13441 T11 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 13442 T11 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 13457 T11 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 13457 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 13458 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 13458 T11 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 13459 T11 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 13460 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 13460 T11 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 13461 T11 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 13462 T11 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 14269 T11 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 14292 T11 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 14301 T11 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 14314 T11 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 14315 T11 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:36536/solr
   [junit4]   2> 14372 T11 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 14373 T11 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 14374 T11 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 14379 T35 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5279effd name:ZooKeeperConnection Watcher:127.0.0.1:36536 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 14380 T11 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 14382 T11 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 14416 T11 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 14419 T38 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@36189ddf name:ZooKeeperConnection Watcher:127.0.0.1:36536/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 14419 T11 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 14441 T11 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 14463 T11 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 14475 T11 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 14480 T11 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 14484 T11 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 14524 T11 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 14527 T11 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59184_tp_hyu%2Fb
   [junit4]   2> 14538 T11 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:59184_tp_hyu%2Fb
   [junit4]   2> 14544 T11 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 14547 T11 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 14550 T11 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 14580 T11 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:59184_tp_hyu%2Fb
   [junit4]   2> 14581 T11 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 14584 T11 oasc.Overseer.start Overseer (id=93437380902060035-127.0.0.1:59184_tp_hyu%2Fb-n_0000000000) starting
   [junit4]   2> 14614 T11 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 14789 T11 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 15532 T40 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 15534 T11 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 15538 T11 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 15541 T11 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 15702 T39 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 16187 T11 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/control-001/cores
   [junit4]   1> CORE DESCRIPTOR: {dataDir=data/, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/control-001/cores/collection1/, name=collection1, config=solrconfig.xml, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/control-001/cores/collection1, loadOnStartup=true, schema=schema.xml, transient=false, coreNodeName=, collection=control_collection, shard=}
   [junit4]   2> 16440 T11 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/control-001/cores/collection1/
   [junit4]   2> 16441 T11 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 16451 T42 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 16451 T42 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 16475 T42 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 16475 T38 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 16486 T39 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59184/tp_hyu/b",
   [junit4]   2> 	  "node_name":"127.0.0.1:59184_tp_hyu%2Fb",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down"} current state version: 0
   [junit4]   2> 16569 T39 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59184/tp_hyu/b",
   [junit4]   2> 	  "node_name":"127.0.0.1:59184_tp_hyu%2Fb",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 16579 T39 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 16647 T39 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 16671 T38 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 17477 T42 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 17477 T42 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 17478 T42 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 17479 T42 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 17480 T42 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 17485 T42 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/control-001/cores/collection1/'
   [junit4]   2> 17527 T42 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 17615 T42 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 17678 T42 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 17805 T42 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 18075 T42 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 18121 T42 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 18268 T42 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 20823 T42 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 20899 T42 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 20902 T42 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 22173 T42 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 22210 T42 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 22218 T42 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 22226 T42 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 22227 T42 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 22227 T42 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 22229 T42 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 22229 T42 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 22253 T42 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 22555 T42 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 22586 T42 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/control-001/cores/collection1/data/
   [junit4]   2> 22599 T42 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5ddbeafc
   [junit4]   2> 22979 T42 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/control-001/cores/collection1/data
   [junit4]   2> 22981 T42 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/control-001/cores/collection1/data/index/
   [junit4]   2> 22982 T42 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/control-001/cores/collection1/data/index' doesn't exist. Creating new index...
   [junit4]   2> 23028 T42 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/control-001/cores/collection1/data/index
   [junit4]   2> 23034 T42 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1705048324, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 23521 T42 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@2bbfe1d8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@42844258; maxCacheMB=0.8662109375 maxMergeSizeMB=0.4140625)),segFN=segments_1,generation=1}
   [junit4]   2> 23527 T42 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 25370 T42 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 25570 T42 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 25603 T42 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 25603 T42 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 25604 T42 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 25604 T42 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 25815 T42 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 25816 T42 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 25816 T42 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 25975 T42 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 25976 T42 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 25978 T42 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 25979 T42 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 25981 T42 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 25983 T42 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 26003 T42 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 26015 T42 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 26018 T42 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 26020 T42 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 26022 T42 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 26023 T42 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 26025 T42 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 26027 T42 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 26028 T42 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 26092 T42 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 26094 T42 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 26139 T42 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 26469 T42 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 26508 T42 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 26511 T42 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 26513 T42 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 26523 T42 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 26650 T42 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 26651 T42 oasu.UpdateLog.init Initializing UpdateLog: dataDir= defaultSyncLevel=FLUSH numRecordsToKeep=100 maxNumLogsToKeep=10
   [junit4]   2> 26703 T42 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 26703 T42 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 26705 T42 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=24, maxMergeAtOnceExplicit=16, maxMergedSegmentMB=88.0419921875, floorSegmentMB=0.5224609375, forceMergeDeletesPctAllowed=22.179663547747115, segmentsPerTier=27.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.5765517444360377
   [junit4]   2> 26707 T42 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@2bbfe1d8 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@42844258; maxCacheMB=0.8662109375 maxMergeSizeMB=0.4140625)),segFN=segments_1,generation=1}
   [junit4]   2> 26708 T42 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 26882 T42 oass.SolrIndexSearcher.<init> Opening Searcher@53c795b4[collection1] main
   [junit4]   2> 26949 T42 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 26952 T42 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 26952 T42 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 26979 T42 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 26979 T42 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 27178 T42 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 27180 T42 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 27181 T42 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 27182 T42 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 27465 T42 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 27476 T43 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@53c795b4[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 27484 T42 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 27492 T46 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:59184/tp_hyu/b collection:control_collection shard:shard1
   [junit4]   2> 27492 T11 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 27493 T11 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 27536 T46 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 27548 T46 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 27551 T38 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 27552 T46 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 27553 T46 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 27553 T39 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 27579 T38 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> ASYNC  NEW_CORE C0 name=collection1 org.apache.solr.core.SolrCore@71ad7c17 url=http://127.0.0.1:59184/tp_hyu/b/collection1 node=127.0.0.1:59184_tp_hyu%2Fb C0_STATE=coll:control_collection core:collection1 props:{core=collection1, base_url=http://127.0.0.1:59184/tp_hyu/b, node_name=127.0.0.1:59184_tp_hyu%2Fb, state=down}
   [junit4]   2> 27581 T46 C0 P59184 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:59184/tp_hyu/b/collection1/
   [junit4]   2> 27589 T46 C0 P59184 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 27589 T46 C0 P59184 oasc.SyncStrategy.syncToMe http://127.0.0.1:59184/tp_hyu/b/collection1/ has no replicas
   [junit4]   2> 27590 T46 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:59184/tp_hyu/b/collection1/ shard1
   [junit4]   2> 27605 T46 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 27613 T38 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 27615 T39 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59184/tp_hyu/b",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 27632 T11 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 27634 T11 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 27636 T50 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2a3998f8 name:ZooKeeperConnection Watcher:127.0.0.1:36536/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 27637 T11 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 27638 T11 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 27640 T11 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 27649 T11 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 27650 T11 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 27650 T11 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 27651 T11 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 27654 T53 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@78c218ac name:ZooKeeperConnection Watcher:127.0.0.1:36536/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 27655 T11 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 27656 T11 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 27660 T38 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 27663 T39 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 27663 T39 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 27665 T38 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 27665 T50 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 27716 T46 oasc.ZkController.register We are http://127.0.0.1:59184/tp_hyu/b/collection1/ and leader is http://127.0.0.1:59184/tp_hyu/b/collection1/
   [junit4]   2> 27727 T46 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:59184/tp_hyu/b
   [junit4]   2> 27728 T46 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 27728 T46 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 27729 T46 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 27731 T38 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 27733 T39 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59184/tp_hyu/b",
   [junit4]   2> 	  "node_name":"127.0.0.1:59184_tp_hyu%2Fb",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active"} current state version: 3
   [junit4]   2> 27735 T39 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59184/tp_hyu/b",
   [junit4]   2> 	  "node_name":"127.0.0.1:59184_tp_hyu%2Fb",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"active"}
   [junit4]   2> 27736 T39 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 27740 T38 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 27740 T50 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 27851 T38 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 27851 T50 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 29551 T11 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/shard-1-001/cores/collection1
   [junit4]   2> 29555 T11 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1 in directory /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/shard-1-001
   [junit4]   2> 29556 T11 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 29559 T11 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:51615
   [junit4]   2> 29560 T11 oascse.JettySolrRunner$1.lifeCycleStarted Jetty properties: {solrconfig=solrconfig.xml, hostPort=50208, solr.data.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/tempDir-001/jetty1, hostContext=/tp_hyu/b, coreRootDirectory=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/shard-1-001/cores}
   [junit4]   2> 29561 T11 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@231bf785
   [junit4]   2> 29562 T11 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/shard-1-001/'
   [junit4]   2> 29611 T11 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/shard-1-001/solr.xml
   [junit4]   2> 29700 T11 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/shard-1-001/cores
   [junit4]   2> 29701 T11 oasc.CoreContainer.<init> New CoreContainer 724509740
   [junit4]   2> 29701 T11 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/shard-1-001/]
   [junit4]   2> 29702 T11 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/shard-1-001/lib
   [junit4]   2> 29704 T11 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/shard-1-001/lib).
   [junit4]   2> 29721 T11 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 29722 T11 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 29722 T11 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 29723 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 29723 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 29724 T11 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 29725 T11 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 29725 T11 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 29726 T11 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 29727 T11 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 29727 T11 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 29729 T11 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 29729 T11 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 29730 T11 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 29731 T11 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 29731 T11 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:36536/solr
   [junit4]   2> 29732 T11 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 29733 T11 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 29734 T11 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 29736 T65 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@14121e2a name:ZooKeeperConnection Watcher:127.0.0.1:36536 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 29737 T11 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 29738 T11 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 29740 T11 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 29742 T68 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4d2f5143 name:ZooKeeperConnection Watcher:127.0.0.1:36536/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 29743 T11 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 29751 T11 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 30761 T11 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:50208_tp_hyu%2Fb
   [junit4]   2> 30762 T11 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:50208_tp_hyu%2Fb
   [junit4]   2> 30769 T11 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 30778 T11 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/shard-1-001/cores
   [junit4]   1> CORE DESCRIPTOR: {transient=false, coreNodeName=, config=solrconfig.xml, name=collection1, schema=schema.xml, loadOnStartup=true, dataDir=data/, collection=collection1, instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/shard-1-001/cores/collection1, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/shard-1-001/cores/collection1/, shard=}
   [junit4]   2> 30781 T11 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/shard-1-001/cores/collection1/
   [junit4]   2> 30781 T11 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 30787 T69 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 30787 T69 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 30791 T38 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 30793 T69 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 30793 T39 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50208/tp_hyu/b",
   [junit4]   2> 	  "node_name":"127.0.0.1:50208_tp_hyu%2Fb",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down"} current state version: 5
   [junit4]   2> 30794 T69 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 30795 T39 oasco.ReplicaMutator.updateState Update state numShards=3 message={
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:50208/tp_hyu/b",
   [junit4]   2> 	  "node_name":"127.0.0.1:50208_tp_hyu%2Fb",
   [junit4]   2> 	  "numShards":"3",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 30796 T69 oascc.ZkStateReader.addZkWatch Updating collection state at /collections/collection1/state.json from ZooKeeper... 
   [junit4]   2> 30796 T39 oasco.ReplicaMutator.updateState Collection already exists with numShards=3
   [junit4]   2> 30797 T39 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard2
   [junit4]   2> 30802 T39 oasco.ZkStateWriter.writePendingUpdates going to update_collection /collections/collection1/state.json version: 0
   [junit4]   2> 30814 T69 oascc.ZkStateReader.updateWatchedCollection Updating data for collection1 to ver 1 
   [junit4]   2> 30814 T69 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 30815 T69 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 30816 T69 oasc.ZkController.createCollectionZkNode Check for collection zkNode:collection1
   [junit4]   2> 30817 T69 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 30817 T69 oascc.ZkStateReader.readConfigName Load collection config from:/collections/collection1
   [junit4]   2> 30819 T69 oascc.ZkStateReader.readConfigName path=/collections/collection1 configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 30820 T69 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/shard-1-001/cores/collection1/'
   [junit4]   2> 30846 T69 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 30850 T69 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 30883 T69 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 30970 T69 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 31066 T69 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 31068 T69 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 31154 T69 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 31697 T69 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 31727 T69 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 31731 T69 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 31787 T69 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 31797 T69 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 31808 T69 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 31811 T69 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 31811 T69 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 31812 T69 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 31814 T69 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 31814 T69 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 31815 T69 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection collection1
   [junit4]   2> 31816 T69 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 31816 T69 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/shard-1-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest C4C1E7A681E74225-001/shard-1-001/cores/collection1/data/
   [junit4]   2> 31817 T69 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@5ddbeafc
   [junit4]   2> 31819 T69 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.

[...truncated too long message...]

rue -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=is -Dtests.timezone=Africa/Luanda -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] ERROR    145s J3 | LeaderFailoverAfterPartitionTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:47947/c8n_1x3_lf_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([C4C1E7A681E74225:4C95D87C2F1B2FDD]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:597)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:920)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:811)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:754)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
   [junit4]    > 	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
   [junit4]    > 	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
   [junit4]    > 	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:47947/c8n_1x3_lf_shard1_replica1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:569)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:211)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:207)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:594)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:461)
   [junit4]    > 	... 54 more
   [junit4]   2> 2037027 T7954 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.LeaderFailoverAfterPartitionTest C4C1E7A681E74225-001
   [junit4]   2> 145259 T7953 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=FastCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=FAST, chunkSize=3597, maxDocsPerChunk=3, blockSize=2), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=FAST, chunkSize=3597, blockSize=2)), sim=RandomSimilarityProvider(queryNorm=false,coord=crazy): {}, locale=is, timezone=Africa/Luanda
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=170967168,total=436207616
   [junit4]   2> NOTE: All tests run in this JVM: [OutOfBoxZkACLAndCredentialsProvidersTest, TestDefaultSearchFieldResource, TestSolrQueryParserResource, AliasIntegrationTest, TestWriterPerf, HdfsCollectionsAPIDistributedZkTest, DistanceUnitsTest, TestUtils, ExitableDirectoryReaderTest, TestJoin, TestIntervalFaceting, HdfsUnloadDistributedZkTest, JsonLoaderTest, TestStressUserVersions, TestJmxMonitoredMap, TestDefaultSimilarityFactory, TestTolerantSearch, TestHighFrequencyDictionaryFactory, ExternalCollectionsTest, TestDynamicLoading, BlockDirectoryTest, TestManagedSchema, CursorMarkTest, SolrCoreTest, TestGroupingSearch, SolrXmlInZkTest, LeaderElectionTest, SoftAutoCommitTest, TestArbitraryIndexDir, DocValuesTest, ShardSplitTest, TestSchemaNameResource, CSVRequestHandlerTest, TestDistributedGrouping, SuggesterWFSTTest, TestBulkSchemaAPI, TestCloudPivotFacet, TestHighlightDedupGrouping, TestJettySolrRunner, TestValueSourceCache, TestAnalyzeInfixSuggestions, TestWordDelimiterFilterFactory, TestLeaderElectionZkExpiry, ShardRoutingTest, BinaryUpdateRequestHandlerTest, TestPseudoReturnFields, OverseerStatusTest, NumericFieldsTest, TestSystemIdResolver, TestIndexingPerformance, TestShardHandlerFactory, CursorPagingTest, UnloadDistributedZkTest, DistributedDebugComponentTest, ResponseLogComponentTest, TermVectorComponentDistributedTest, DocValuesMultiTest, DefaultValueUpdateProcessorTest, CollectionsAPIDistributedZkTest, TestHashPartitioner, TestRealTimeGet, TestStressVersions, TestRangeQuery, TestCoreContainer, PeerSyncTest, BasicFunctionalityTest, TestIndexSearcher, CurrencyFieldXmlFileTest, SignatureUpdateProcessorFactoryTest, SuggesterFSTTest, TestFoldingMultitermQuery, SpatialFilterTest, SolrCoreCheckLockOnStartupTest, TestSolrDeletionPolicy1, TestSurroundQueryParser, StandardRequestHandlerTest, TestQueryTypes, RequestHandlersTest, FieldAnalysisRequestHandlerTest, IndexSchemaRuntimeFieldTest, SolrPluginUtilsTest, TestCollationField, UpdateRequestProcessorFactoryTest, TestAnalyzedSuggestions, SOLR749Test, TestQuerySenderNoQuery, TestSolrDeletionPolicy2, SampleTest, TestSearchPerf, OutputWriterTest, TestPostingsSolrHighlighter, SpellPossibilityIteratorTest, SynonymTokenizerTest, EchoParamsTest, TestIBSimilarityFactory, TestFastOutputStream, PluginInfoTest, DateMathParserTest, TestSuggestSpellingConverter, SpellingQueryConverterTest, DOMUtilTest, ClusterStateTest, TestSolrJ, TestLRUCache, FileUtilsTest, DistributedMLTComponentTest, TestSimpleTrackingShardHandler, CollectionsAPIAsyncDistributedZkTest, DeleteLastCustomShardedReplicaTest, DeleteShardTest, DistribDocExpirationUpdateProcessorTest, LeaderFailoverAfterPartitionTest]
   [junit4] Completed on J3 in 146.05s, 1 test, 1 error <<< FAILURES!

[...truncated 399 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 470 suites, 1873 tests, 5 errors, 58 ignored (9 assumptions)

Total time: 162 minutes 25 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 754445131 bytes
Compression is 0.0%
Took 2 min 21 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 778 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/778/

8 tests failed.
FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
ERROR: SolrIndexSearcher opens=30 closes=29

Stack Trace:
java.lang.AssertionError: ERROR: SolrIndexSearcher opens=30 closes=29
	at __randomizedtesting.SeedInfo.seed([5B237F1A3DCE8604]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.SolrTestCaseJ4.endTrackingSearchers(SolrTestCaseJ4.java:494)
	at org.apache.solr.SolrTestCaseJ4.afterClass(SolrTestCaseJ4.java:230)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:799)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  junit.framework.TestSuite.org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest

Error Message:
2 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest:     1) Thread[id=84, name=qtp805310106-84, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at java.lang.Object.wait(Native Method)         at java.lang.Object.wait(Object.java:503)         at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.blockUntilFinished(ConcurrentUpdateSolrClient.java:389)         at org.apache.solr.update.StreamingSolrClients.blockUntilFinished(StreamingSolrClients.java:103)         at org.apache.solr.update.SolrCmdDistributor.blockAndDoRetries(SolrCmdDistributor.java:229)         at org.apache.solr.update.SolrCmdDistributor.finish(SolrCmdDistributor.java:89)         at org.apache.solr.update.processor.DistributedUpdateProcessor.doFinish(DistributedUpdateProcessor.java:780)         at org.apache.solr.update.processor.DistributedUpdateProcessor.finish(DistributedUpdateProcessor.java:1630)         at org.apache.solr.update.processor.LogUpdateProcessor.finish(LogUpdateProcessorFactory.java:183)         at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:83)         at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:144)         at org.apache.solr.core.SolrCore.execute(SolrCore.java:2026)         at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:808)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:435)         at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:218)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)         at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:102)         at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)         at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)         at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)         at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)         at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)         at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)         at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)         at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)         at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)         at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)         at org.eclipse.jetty.server.Server.handle(Server.java:368)         at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)         at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)         at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:953)         at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1014)         at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:953)         at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)         at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)         at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)         at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)         at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)         at java.lang.Thread.run(Thread.java:745)    2) Thread[id=95, name=searcherExecutor-39-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]         at sun.misc.Unsafe.park(Native Method)         at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)         at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)         at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)         at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)         at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)         at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)         at java.lang.Thread.run(Thread.java:745)

Stack Trace:
com.carrotsearch.randomizedtesting.ThreadLeakError: 2 threads leaked from SUITE scope at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest: 
   1) Thread[id=84, name=qtp805310106-84, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at java.lang.Object.wait(Native Method)
        at java.lang.Object.wait(Object.java:503)
        at org.apache.solr.client.solrj.impl.ConcurrentUpdateSolrClient.blockUntilFinished(ConcurrentUpdateSolrClient.java:389)
        at org.apache.solr.update.StreamingSolrClients.blockUntilFinished(StreamingSolrClients.java:103)
        at org.apache.solr.update.SolrCmdDistributor.blockAndDoRetries(SolrCmdDistributor.java:229)
        at org.apache.solr.update.SolrCmdDistributor.finish(SolrCmdDistributor.java:89)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.doFinish(DistributedUpdateProcessor.java:780)
        at org.apache.solr.update.processor.DistributedUpdateProcessor.finish(DistributedUpdateProcessor.java:1630)
        at org.apache.solr.update.processor.LogUpdateProcessor.finish(LogUpdateProcessorFactory.java:183)
        at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:83)
        at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:144)
        at org.apache.solr.core.SolrCore.execute(SolrCore.java:2026)
        at org.apache.solr.servlet.SolrDispatchFilter.execute(SolrDispatchFilter.java:808)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:435)
        at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:218)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
        at org.apache.solr.client.solrj.embedded.JettySolrRunner$DebugFilter.doFilter(JettySolrRunner.java:102)
        at org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1419)
        at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:455)
        at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:229)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
        at org.eclipse.jetty.server.handler.GzipHandler.handle(GzipHandler.java:301)
        at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1077)
        at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:384)
        at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
        at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1009)
        at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
        at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
        at org.eclipse.jetty.server.Server.handle(Server.java:368)
        at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:489)
        at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
        at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:953)
        at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1014)
        at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:953)
        at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)
        at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
        at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
        at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
        at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
        at java.lang.Thread.run(Thread.java:745)
   2) Thread[id=95, name=searcherExecutor-39-thread-1, state=WAITING, group=TGRP-ChaosMonkeyNothingIsSafeTest]
        at sun.misc.Unsafe.park(Native Method)
        at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
        at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
        at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442)
        at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1068)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1130)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
	at __randomizedtesting.SeedInfo.seed([5B237F1A3DCE8604]:0)


REGRESSION:  org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds

Error Message:
soft529 wasn't fast enough

Stack Trace:
java.lang.AssertionError: soft529 wasn't fast enough
	at __randomizedtesting.SeedInfo.seed([5B237F1A3DCE8604:AF7869A8CBDB6A3]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertNotNull(Assert.java:526)
	at org.apache.solr.update.SoftAutoCommitTest.testSoftAndHardCommitMaxTimeMixedAdds(SoftAutoCommitTest.java:111)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:25297/tht/dc/c8n_1x2_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:25297/tht/dc/c8n_1x2_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([5B237F1A3DCE8604:D37740C09332EBFC]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:597)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:920)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:811)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:754)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:25297/tht/dc/c8n_1x2_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:569)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:211)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:207)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:594)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:461)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:20658/c8n_1x3_lf_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:20658/c8n_1x3_lf_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([5B237F1A3DCE8604:D37740C09332EBFC]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:597)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:920)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:811)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:754)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:20658/c8n_1x3_lf_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:569)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:211)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:207)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:594)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:461)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:33393/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33393/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([5B237F1A3DCE8604:D37740C09332EBFC]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:569)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:211)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:207)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:461)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22065/wnt/gq/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22065/wnt/gq/collection1
	at __randomizedtesting.SeedInfo.seed([5B237F1A3DCE8604:D37740C09332EBFC]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:597)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:920)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:811)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:754)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22065/wnt/gq/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:569)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:211)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:207)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:594)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:461)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22118/_b/repfacttest_c8n_1x3_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22118/_b/repfacttest_c8n_1x3_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([5B237F1A3DCE8604:D37740C09332EBFC]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:597)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:920)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:811)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:754)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:22118/_b/repfacttest_c8n_1x3_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:569)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:211)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:207)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:594)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:461)
	... 53 more




Build Log:
[...truncated 10079 lines...]
   [junit4] Suite: org.apache.solr.update.SoftAutoCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 5B237F1A3DCE8604-001/init-core-data-001
   [junit4]   2> 32300 T62 oas.SolrTestCaseJ4.buildSSLConfig Randomized ssl (false) and clientAuth (false)
   [junit4]   2> 32302 T62 oas.SolrTestCaseJ4.initCore ####initCore
   [junit4]   2> 32303 T62 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 32305 T62 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 32306 T62 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 32307 T62 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 32507 T62 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 32582 T62 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 32659 T62 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 32660 T62 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 32740 T62 oass.IndexSchema.readSchema [null] Schema name=test
   [junit4]   2> 33256 T62 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 33275 T62 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 33278 T62 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 33305 T62 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 33312 T62 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 33317 T62 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 33318 T62 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 33319 T62 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 33319 T62 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 33320 T62 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 33320 T62 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 33338 T62 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 33338 T62 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr
   [junit4]   2> 33339 T62 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/'
   [junit4]   2> 33371 T62 oasc.CoreContainer.<init> New CoreContainer 2068215434
   [junit4]   2> 33372 T62 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/]
   [junit4]   2> 33372 T62 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/lib
   [junit4]   2> 33373 T62 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/lib).
   [junit4]   2> 33396 T62 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 600000
   [junit4]   2> 33397 T62 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: null
   [junit4]   2> 33397 T62 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 60000
   [junit4]   2> 33397 T62 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 33398 T62 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 33398 T62 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 33398 T62 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 33399 T62 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 33399 T62 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 33399 T62 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 33400 T62 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 33401 T62 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=30000&connTimeout=30000&retry=true
   [junit4]   2> 33401 T62 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 33402 T62 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 33402 T62 oasc.CoreContainer.load Node Name: testNode
   [junit4]   1> CORE DESCRIPTOR: {instanceDir=collection1, shard=shard1, transient=false, config=solrconfig.xml, absoluteInstDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/, name=collection1, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 5B237F1A3DCE8604-001/init-core-data-001, collection=collection1, schema=schema.xml, loadOnStartup=true}
   [junit4]   2> 33411 T63 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 33414 T63 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 33415 T63 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 33416 T63 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 33576 T63 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 33627 T63 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 33725 T63 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 33727 T63 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 33815 T63 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 34441 T63 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 34467 T63 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 34471 T63 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 34508 T63 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 34517 T63 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 34524 T63 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 34525 T63 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 34525 T63 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 34526 T63 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 34527 T63 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 34528 T63 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 34528 T63 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 34529 T63 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 34530 T63 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 5B237F1A3DCE8604-001/init-core-data-001/
   [junit4]   2> 34530 T63 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@116639
   [junit4]   2> 34532 T63 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 5B237F1A3DCE8604-001/init-core-data-001
   [junit4]   2> 34532 T63 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 5B237F1A3DCE8604-001/init-core-data-001/index/
   [junit4]   2> 34533 T63 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 5B237F1A3DCE8604-001/init-core-data-001/index' doesn't exist. Creating new index...
   [junit4]   2> 34534 T63 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 5B237F1A3DCE8604-001/init-core-data-001/index
   [junit4]   2> 34540 T63 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@3c0c7b82
   [junit4]   2> 34543 T63 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@424dc957 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@dd90c3; maxCacheMB=0.552734375 maxMergeSizeMB=0.2451171875)),segFN=segments_1,generation=1}
   [junit4]   2> 34543 T63 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 34544 T63 oasc.SolrCore.initWriters created xml: solr.XMLResponseWriter
   [junit4]   2> 34555 T63 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 34555 T63 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 34556 T63 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 34557 T63 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 34557 T63 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 34558 T63 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 34558 T63 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 34559 T63 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 34559 T63 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 34560 T63 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 34561 T63 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 34562 T63 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 34564 T63 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 34565 T63 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 34566 T63 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 34567 T63 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 34569 T63 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 34570 T63 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 34571 T63 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 34572 T63 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 34574 T63 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 34576 T63 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 34577 T63 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 34578 T63 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 34579 T63 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 34580 T63 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 34582 T63 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 34583 T63 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 34584 T63 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 34589 T63 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 34591 T63 oasc.RequestHandlers.initHandlersFromConfig created dismax: solr.SearchHandler
   [junit4]   2> 34593 T63 oasc.RequestHandlers.initHandlersFromConfig created mock: org.apache.solr.core.MockQuerySenderListenerReqHandler
   [junit4]   2> 34595 T63 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 34595 T63 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 34596 T63 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 34597 T63 oasc.RequestHandlers.initHandlersFromConfig created /terms: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 34598 T63 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 34599 T63 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH_Direct: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 34601 T63 oasc.RequestHandlers.initHandlersFromConfig created spellCheckWithWordbreak: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 34602 T63 oasc.RequestHandlers.initHandlersFromConfig created spellCheckWithWordbreak_Direct: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 34604 T63 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH1: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 34605 T63 oasc.RequestHandlers.initHandlersFromConfig created mltrh: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 34606 T63 oasc.RequestHandlers.initHandlersFromConfig created tvrh: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 34607 T63 oasc.RequestHandlers.initHandlersFromConfig created /mlt: solr.MoreLikeThisHandler
   [junit4]   2> 34608 T63 oasc.RequestHandlers.initHandlersFromConfig created /debug/dump: solr.DumpRequestHandler
   [junit4]   2> 34609 T63 oasc.RequestHandlers.initHandlersFromConfig created /search-facet-def: solr.SearchHandler
   [junit4]   2> 34610 T63 oasc.RequestHandlers.initHandlersFromConfig created /search-facet-invariants: solr.SearchHandler
   [junit4]   2> 34647 T63 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 34650 T63 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 34653 T63 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 34655 T63 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 34658 T63 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 34662 T63 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 34664 T63 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 34665 T63 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 34665 T63 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 34667 T63 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=38, maxMergeAtOnceExplicit=42, maxMergedSegmentMB=12.2060546875, floorSegmentMB=1.3203125, forceMergeDeletesPctAllowed=10.802641641070448, segmentsPerTier=29.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.0
   [junit4]   2> 34668 T63 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(NRTCachingDirectory(RAMDirectory@424dc957 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@dd90c3; maxCacheMB=0.552734375 maxMergeSizeMB=0.2451171875)),segFN=segments_1,generation=1}
   [junit4]   2> 34669 T63 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 34670 T63 oass.SolrIndexSearcher.<init> Opening Searcher@19e9f0e8[collection1] main
   [junit4]   2> 34671 T63 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 34671 T63 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 34672 T63 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 34672 T63 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 34693 T63 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 34759 T63 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 34760 T64 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 34762 T64 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 34762 T64 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 34763 T64 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 34764 T64 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 34766 T63 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 34766 T64 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 34768 T64 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 34769 T64 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 34769 T62 oas.SolrTestCaseJ4.initCore ####initCore end
   [junit4]   2> 34771 T64 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 34775 T64 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@19e9f0e8[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 34782 T62 oas.SolrTestCaseJ4.setUp ###Starting testSoftAndHardCommitMaxTimeMixedAdds
   [junit4]   2> 34783 T62 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/'
   [junit4]   2> 34784 T62 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/classes/' to classloader
   [junit4]   2> 34785 T62 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/.svn/' to classloader
   [junit4]   2> 34786 T62 oasc.SolrResourceLoader.replaceClassLoader Adding 'file:/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/lib/README' to classloader
   [junit4]   2> 34978 T62 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 35029 T62 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 35126 T62 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 35128 T62 oass.IndexSchema.readSchema Reading Solr Schema from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml
   [junit4]   2> 35215 T62 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 35727 T62 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 35744 T62 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 35746 T62 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 35769 T62 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 35774 T62 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 35778 T62 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 35779 T62 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 35780 T62 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 35780 T62 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 35781 T62 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 35781 T62 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 35782 T62 oasc.CoreContainer.reload Reloading SolrCore 'collection1' using configuration from instancedir /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/
   [junit4]   2> 35783 T62 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 5B237F1A3DCE8604-001/init-core-data-001/
   [junit4]   2> 35783 T62 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@116639
   [junit4]   2> 35784 T62 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.update.SoftAutoCommitTest 5B237F1A3DCE8604-001/init-core-data-001/index/
   [junit4]   2> 35785 T62 oasc.SolrCore.initWriters created xml: solr.XMLResponseWriter
   [junit4]   2> 35795 T62 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 35795 T62 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 35796 T62 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 35796 T62 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe-allfields"
   [junit4]   2> 35797 T62 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 35797 T62 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 35798 T62 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 35799 T62 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "uniq-fields"
   [junit4]   2> 35799 T62 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 35800 T62 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 35800 T62 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 35801 T62 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 35802 T62 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 35803 T62 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 35805 T62 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 35806 T62 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 35807 T62 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 35808 T62 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 35815 T62 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 35816 T62 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 35817 T62 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 35819 T62 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 35820 T62 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 35821 T62 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 35822 T62 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 35823 T62 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 35824 T62 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 35825 T62 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 35827 T62 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 35828 T62 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 35829 T62 oasc.RequestHandlers.initHandlersFromConfig created dismax: solr.SearchHandler
   [junit4]   2> 35830 T62 oasc.RequestHandlers.initHandlersFromConfig created mock: org.apache.solr.core.MockQuerySenderListenerReqHandler
   [junit4]   2> 35831 T62 oasc.RequestHandlers.initHandlersFromConfig created defaults: solr.StandardRequestHandler
   [junit4]   2> 35832 T62 oasc.RequestHandlers.initHandlersFromConfig adding lazy requestHandler: solr.StandardRequestHandler
   [junit4]   2> 35832 T62 oasc.RequestHandlers.initHandlersFromConfig created lazy: solr.StandardRequestHandler
   [junit4]   2> 35833 T62 oasc.RequestHandlers.initHandlersFromConfig created /terms: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 35834 T62 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 35835 T62 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH_Direct: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 35836 T62 oasc.RequestHandlers.initHandlersFromConfig created spellCheckWithWordbreak: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 35838 T62 oasc.RequestHandlers.initHandlersFromConfig created spellCheckWithWordbreak_Direct: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 35839 T62 oasc.RequestHandlers.initHandlersFromConfig created spellCheckCompRH1: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 35840 T62 oasc.RequestHandlers.initHandlersFromConfig created mltrh: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 35841 T62 oasc.RequestHandlers.initHandlersFromConfig created tvrh: org.apache.solr.handler.component.SearchHandler
   [junit4]   2> 35842 T62 oasc.RequestHandlers.initHandlersFromConfig created /mlt: solr.MoreLikeThisHandler
   [junit4]   2> 35843 T62 oasc.RequestHandlers.initHandlersFromConfig created /debug/dump: solr.DumpRequestHandler
   [junit4]   2> 35844 T62 oasc.RequestHandlers.initHandlersFromConfig created /search-facet-def: solr.SearchHandler
   [junit4]   2> 35845 T62 oasc.RequestHandlers.initHandlersFromConfig created /search-facet-invariants: solr.SearchHandler
   [junit4]   2> 35864 T62 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 35866 T62 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 35868 T62 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 35870 T62 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 35872 T62 oasc.SolrCore.initDeprecatedSupport WARN solrconfig.xml uses deprecated <admin/gettableFiles>, Please update your config to use the ShowFileRequestHandler.
   [junit4]   2> 35874 T62 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 35880 T62 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 35880 T62 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 35882 T62 oass.SolrIndexSearcher.<init> Opening Searcher@4cae1343[collection1] main
   [junit4]   2> 35883 T62 oasr.RestManager.init Initializing RestManager with initArgs: {storageIO=org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO}
   [junit4]   2> 35883 T62 oasr.ManagedResourceStorage.load Reading _rest_managed.json using InMemoryStorage
   [junit4]   2> 35884 T62 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 35884 T62 oashc.SpellCheckComponent.inform Initializing spell checkers
   [junit4]   2> 35887 T62 oass.DirectSolrSpellChecker.init init: {name=direct,classname=DirectSolrSpellChecker,field=lowerfilt,minQueryLength=3}
   [junit4]   2> 35899 T62 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 35899 T66 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: default
   [junit4]   2> 35901 T66 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: direct
   [junit4]   2> 35901 T66 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: wordbreak
   [junit4]   2> 35901 T66 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: multipleFields
   [junit4]   2> 35902 T66 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: jarowinkler
   [junit4]   2> 35903 T62 oasu.DefaultSolrCoreState.newIndexWriter Creating new IndexWriter...
   [junit4]   2> 35903 T66 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: external
   [junit4]   2> 35904 T62 oasu.DefaultSolrCoreState.newIndexWriter Waiting until IndexWriter is unused... core=collection1
   [junit4]   2> 35904 T62 oasu.DefaultSolrCoreState.newIndexWriter Closing old IndexWriter... core=collection1
   [junit4]   2> 35905 T66 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: freq
   [junit4]   2> 35905 T62 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogDocMergePolicy: [LogDocMergePolicy: minMergeSize=1000, mergeFactor=10, maxMergeSize=9223372036854775807, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0]
   [junit4]   2> 35906 T66 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: fqcn
   [junit4]   2> 35907 T66 oashc.SpellCheckComponent$SpellCheckerListener.newSearcher Loading spell index for spellchecker: perDict
   [junit4]   2> 35911 T66 oasc.SolrCore.re

[...truncated too long message...]

d4LetterWord connecting to 127.0.0.1:34028 34028
   [junit4]   2> 2855421 T8265 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		5	/solr/clusterstate.json
   [junit4]   2> 		5	/solr/aliases.json
   [junit4]   2> 		4	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		5	/solr/live_nodes
   [junit4]   2> 		4	/solr/overseer/queue
   [junit4]   2> 		4	/solr/overseer/collection-queue-work
   [junit4]   2> 	
   [junit4]   2> 2855422 T8264 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:55863/
   [junit4]   2> 2855422 T8264 oasc.SocketProxy.close WARN Closing 11 connections to: http://127.0.0.1:20658/
   [junit4]   2> 2855422 T8264 oasc.SocketProxy.close WARN Closing 5 connections to: http://127.0.0.1:12621/
   [junit4]   2> 2855423 T8264 oasc.SocketProxy.close WARN Closing 6 connections to: http://127.0.0.1:15195/
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=LeaderFailoverAfterPartitionTest -Dtests.method=test -Dtests.seed=5B237F1A3DCE8604 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=et -Dtests.timezone=America/Hermosillo -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR    140s J2 | LeaderFailoverAfterPartitionTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:20658/c8n_1x3_lf_shard1_replica1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([5B237F1A3DCE8604:D37740C09332EBFC]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:597)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:920)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:811)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:754)
   [junit4]    > 	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
   [junit4]    > 	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
   [junit4]    > 	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
   [junit4]    > 	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
   [junit4]    > 	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:20658/c8n_1x3_lf_shard1_replica1
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:569)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:211)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:207)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:594)
   [junit4]    > 	... 49 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:461)
   [junit4]    > 	... 54 more
   [junit4]   2> 2855437 T8264 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.LeaderFailoverAfterPartitionTest 5B237F1A3DCE8604-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {intDefault=FSTOrd50, range_facet_l=PostingsFormat(name=MockRandom), id=PostingsFormat(name=LuceneVarGapDocFreqInterval), timestamp=PostingsFormat(name=MockRandom), text=PostingsFormat(name=LuceneVarGapFixedInterval), multiDefault=PostingsFormat(name=MockRandom), _version_=FSTOrd50, a_t=PostingsFormat(name=MockRandom)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=DefaultSimilarity, locale=et, timezone=America/Hermosillo
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=54222424,total=350748672
   [junit4]   2> NOTE: All tests run in this JVM: [SolrCoreCheckLockOnStartupTest, SharedFSAutoReplicaFailoverUtilsTest, ChaosMonkeyNothingIsSafeTest, TestReversedWildcardFilterFactory, TestSolrConfigHandler, BadCopyFieldTest, AddBlockUpdateTest, TestInitParams, TestRandomDVFaceting, TestDistribDocBasedVersion, DistribCursorPagingTest, HdfsBasicDistributedZkTest, DistribJoinFromCollectionTest, RollingRestartTest, StandardRequestHandlerTest, TestCSVResponseWriter, CachingDirectoryFactoryTest, SolrIndexSplitterTest, TestNoOpRegenerator, MergeStrategyTest, TestClassNameShortening, TestFastWriter, TestCloudSchemaless, TestManagedSchemaDynamicFieldResource, TestReqParamsAPI, TestSuggestSpellingConverter, TestReloadAndDeleteDocs, BasicDistributedZk2Test, TestAnalyzedSuggestions, TestReload, TestBinaryField, TestQuerySenderNoQuery, LukeRequestHandlerTest, TestExceedMaxTermLength, FileBasedSpellCheckerTest, RAMDirectoryFactoryTest, TestLuceneMatchVersion, TestBM25SimilarityFactory, AnalyticsMergeStrategyTest, TestReRankQParserPlugin, HdfsRecoveryZkTest, TestHdfsUpdateLog, CurrencyFieldOpenExchangeTest, SaslZkACLProviderTest, AnalyticsQueryTest, SpatialRPTFieldTypeTest, VMParamsZkACLAndCredentialsProvidersTest, LeaderFailoverAfterPartitionTest]
   [junit4] Completed on J2 in 140.12s, 1 test, 1 error <<< FAILURES!

[...truncated 50 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 470 suites, 1873 tests, 2 suite-level errors, 5 errors, 1 failure, 58 ignored (9 assumptions)

Total time: 184 minutes 38 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 654976551 bytes
Compression is 0.0%
Took 1 min 59 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 777 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/777/

8 tests failed.
REGRESSION:  org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test

Error Message:
There were too many update fails - we expect it can happen, but shouldn't easily

Stack Trace:
java.lang.AssertionError: There were too many update fails - we expect it can happen, but shouldn't easily
	at __randomizedtesting.SeedInfo.seed([6A03F0A0A3FD6B7D:E257CF7A0D010685]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.junit.Assert.assertFalse(Assert.java:68)
	at org.apache.solr.cloud.ChaosMonkeyNothingIsSafeTest.test(ChaosMonkeyNothingIsSafeTest.java:222)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test

Error Message:
shard1 is not consistent.  Got 305 from https://127.0.0.1:27817/collection1lastClient and got 283 from https://127.0.0.1:37517/collection1

Stack Trace:
java.lang.AssertionError: shard1 is not consistent.  Got 305 from https://127.0.0.1:27817/collection1lastClient and got 283 from https://127.0.0.1:37517/collection1
	at __randomizedtesting.SeedInfo.seed([6A03F0A0A3FD6B7D:E257CF7A0D010685]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1286)
	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1265)
	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:161)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload

Error Message:
expected:<[{indexVersion=1425569204107,generation=2,filelist=[_1h0i.fdt, _1h0i.fdx, _1h0i.fnm, _1h0i.nvd, _1h0i.nvm, _1h0i.si, _1h0i_LuceneVarGapDocFreqInterval_0.doc, _1h0i_LuceneVarGapDocFreqInterval_0.tib, _1h0i_LuceneVarGapDocFreqInterval_0.tiv, _1h2a.fdt, _1h2a.fdx, _1h2a.fnm, _1h2a.nvd, _1h2a.nvm, _1h2a.si, _1h2a_LuceneVarGapDocFreqInterval_0.doc, _1h2a_LuceneVarGapDocFreqInterval_0.tib, _1h2a_LuceneVarGapDocFreqInterval_0.tiv, _1h2b.cfe, _1h2b.cfs, _1h2b.si, _1h2c.fdt, _1h2c.fdx, _1h2c.fnm, _1h2c.nvd, _1h2c.nvm, _1h2c.si, _1h2c_LuceneVarGapDocFreqInterval_0.doc, _1h2c_LuceneVarGapDocFreqInterval_0.tib, _1h2c_LuceneVarGapDocFreqInterval_0.tiv, _1h2d.fdt, _1h2d.fdx, _1h2d.fnm, _1h2d.nvd, _1h2d.nvm, _1h2d.si, _1h2d_LuceneVarGapDocFreqInterval_0.doc, _1h2d_LuceneVarGapDocFreqInterval_0.tib, _1h2d_LuceneVarGapDocFreqInterval_0.tiv, _1h2e.fdt, _1h2e.fdx, _1h2e.fnm, _1h2e.nvd, _1h2e.nvm, _1h2e.si, _1h2e_LuceneVarGapDocFreqInterval_0.doc, _1h2e_LuceneVarGapDocFreqInterval_0.tib, _1h2e_LuceneVarGapDocFreqInterval_0.tiv, _1h2f.fdt, _1h2f.fdx, _1h2f.fnm, _1h2f.nvd, _1h2f.nvm, _1h2f.si, _1h2f_LuceneVarGapDocFreqInterval_0.doc, _1h2f_LuceneVarGapDocFreqInterval_0.tib, _1h2f_LuceneVarGapDocFreqInterval_0.tiv, _1h2g.fdt, _1h2g.fdx, _1h2g.fnm, _1h2g.nvd, _1h2g.nvm, _1h2g.si, _1h2g_LuceneVarGapDocFreqInterval_0.doc, _1h2g_LuceneVarGapDocFreqInterval_0.tib, _1h2g_LuceneVarGapDocFreqInterval_0.tiv, _1h2h.fdt, _1h2h.fdx, _1h2h.fnm, _1h2h.nvd, _1h2h.nvm, _1h2h.si, _1h2h_LuceneVarGapDocFreqInterval_0.doc, _1h2h_LuceneVarGapDocFreqInterval_0.tib, _1h2h_LuceneVarGapDocFreqInterval_0.tiv, _1h2i.fdt, _1h2i.fdx, _1h2i.fnm, _1h2i.nvd, _1h2i.nvm, _1h2i.si, _1h2i_LuceneVarGapDocFreqInterval_0.doc, _1h2i_LuceneVarGapDocFreqInterval_0.tib, _1h2i_LuceneVarGapDocFreqInterval_0.tiv, _1h2j.fdt, _1h2j.fdx, _1h2j.fnm, _1h2j.nvd, _1h2j.nvm, _1h2j.si, _1h2j_LuceneVarGapDocFreqInterval_0.doc, _1h2j_LuceneVarGapDocFreqInterval_0.tib, _1h2j_LuceneVarGapDocFreqInterval_0.tiv, _1h2k.fdt, _1h2k.fdx, _1h2k.fnm, _1h2k.nvd, _1h2k.nvm, _1h2k.si, _1h2k_LuceneVarGapDocFreqInterval_0.doc, _1h2k_LuceneVarGapDocFreqInterval_0.tib, _1h2k_LuceneVarGapDocFreqInterval_0.tiv, _1h2l.fdt, _1h2l.fdx, _1h2l.fnm, _1h2l.nvd, _1h2l.nvm, _1h2l.si, _1h2l_LuceneVarGapDocFreqInterval_0.doc, _1h2l_LuceneVarGapDocFreqInterval_0.tib, _1h2l_LuceneVarGapDocFreqInterval_0.tiv, _1h2m.fdt, _1h2m.fdx, _1h2m.fnm, _1h2m.nvd, _1h2m.nvm, _1h2m.si, _1h2m_LuceneVarGapDocFreqInterval_0.doc, _1h2m_LuceneVarGapDocFreqInterval_0.tib, _1h2m_LuceneVarGapDocFreqInterval_0.tiv, _1h2n.fdt, _1h2n.fdx, _1h2n.fnm, _1h2n.nvd, _1h2n.nvm, _1h2n.si, _1h2n_LuceneVarGapDocFreqInterval_0.doc, _1h2n_LuceneVarGapDocFreqInterval_0.tib, _1h2n_LuceneVarGapDocFreqInterval_0.tiv, _1h2o.fdt, _1h2o.fdx, _1h2o.fnm, _1h2o.nvd, _1h2o.nvm, _1h2o.si, _1h2o_LuceneVarGapDocFreqInterval_0.doc, _1h2o_LuceneVarGapDocFreqInterval_0.tib, _1h2o_LuceneVarGapDocFreqInterval_0.tiv, _1h2p.fdt, _1h2p.fdx, _1h2p.fnm, _1h2p.nvd, _1h2p.nvm, _1h2p.si, _1h2p_LuceneVarGapDocFreqInterval_0.doc, _1h2p_LuceneVarGapDocFreqInterval_0.tib, _1h2p_LuceneVarGapDocFreqInterval_0.tiv, _1h2q.fdt, _1h2q.fdx, _1h2q.fnm, _1h2q.nvd, _1h2q.nvm, _1h2q.si, _1h2q_LuceneVarGapDocFreqInterval_0.doc, _1h2q_LuceneVarGapDocFreqInterval_0.tib, _1h2q_LuceneVarGapDocFreqInterval_0.tiv, _1h2r.fdt, _1h2r.fdx, _1h2r.fnm, _1h2r.nvd, _1h2r.nvm, _1h2r.si, _1h2r_LuceneVarGapDocFreqInterval_0.doc, _1h2r_LuceneVarGapDocFreqInterval_0.tib, _1h2r_LuceneVarGapDocFreqInterval_0.tiv, _1h2s.fdt, _1h2s.fdx, _1h2s.fnm, _1h2s.nvd, _1h2s.nvm, _1h2s.si, _1h2s_LuceneVarGapDocFreqInterval_0.doc, _1h2s_LuceneVarGapDocFreqInterval_0.tib, _1h2s_LuceneVarGapDocFreqInterval_0.tiv, _1h2t.fdt, _1h2t.fdx, _1h2t.fnm, _1h2t.nvd, _1h2t.nvm, _1h2t.si, _1h2t_LuceneVarGapDocFreqInterval_0.doc, _1h2t_LuceneVarGapDocFreqInterval_0.tib, _1h2t_LuceneVarGapDocFreqInterval_0.tiv, _1h2u.fdt, _1h2u.fdx, _1h2u.fnm, _1h2u.nvd, _1h2u.nvm, _1h2u.si, _1h2u_LuceneVarGapDocFreqInterval_0.doc, _1h2u_LuceneVarGapDocFreqInterval_0.tib, _1h2u_LuceneVarGapDocFreqInterval_0.tiv, _1h2v.fdt, _1h2v.fdx, _1h2v.fnm, _1h2v.nvd, _1h2v.nvm, _1h2v.si, _1h2v_LuceneVarGapDocFreqInterval_0.doc, _1h2v_LuceneVarGapDocFreqInterval_0.tib, _1h2v_LuceneVarGapDocFreqInterval_0.tiv, _1h2w.fdt, _1h2w.fdx, _1h2w.fnm, _1h2w.nvd, _1h2w.nvm, _1h2w.si, _1h2w_LuceneVarGapDocFreqInterval_0.doc, _1h2w_LuceneVarGapDocFreqInterval_0.tib, _1h2w_LuceneVarGapDocFreqInterval_0.tiv, _1h2x.fdt, _1h2x.fdx, _1h2x.fnm, _1h2x.nvd, _1h2x.nvm, _1h2x.si, _1h2x_LuceneVarGapDocFreqInterval_0.doc, _1h2x_LuceneVarGapDocFreqInterval_0.tib, _1h2x_LuceneVarGapDocFreqInterval_0.tiv, _1h2y.fdt, _1h2y.fdx, _1h2y.fnm, _1h2y.nvd, _1h2y.nvm, _1h2y.si, _1h2y_LuceneVarGapDocFreqInterval_0.doc, _1h2y_LuceneVarGapDocFreqInterval_0.tib, _1h2y_LuceneVarGapDocFreqInterval_0.tiv, _1h2z.fdt, _1h2z.fdx, _1h2z.fnm, _1h2z.nvd, _1h2z.nvm, _1h2z.si, _1h2z_LuceneVarGapDocFreqInterval_0.doc, _1h2z_LuceneVarGapDocFreqInterval_0.tib, _1h2z_LuceneVarGapDocFreqInterval_0.tiv, _1h30.fdt, _1h30.fdx, _1h30.fnm, _1h30.nvd, _1h30.nvm, _1h30.si, _1h30_LuceneVarGapDocFreqInterval_0.doc, _1h30_LuceneVarGapDocFreqInterval_0.tib, _1h30_LuceneVarGapDocFreqInterval_0.tiv, _1h31.fdt, _1h31.fdx, _1h31.fnm, _1h31.nvd, _1h31.nvm, _1h31.si, _1h31_LuceneVarGapDocFreqInterval_0.doc, _1h31_LuceneVarGapDocFreqInterval_0.tib, _1h31_LuceneVarGapDocFreqInterval_0.tiv, _1h32.fdt, _1h32.fdx, _1h32.fnm, _1h32.nvd, _1h32.nvm, _1h32.si, _1h32_LuceneVarGapDocFreqInterval_0.doc, _1h32_LuceneVarGapDocFreqInterval_0.tib, _1h32_LuceneVarGapDocFreqInterval_0.tiv, _1h33.fdt, _1h33.fdx, _1h33.fnm, _1h33.nvd, _1h33.nvm, _1h33.si, _1h33_LuceneVarGapDocFreqInterval_0.doc, _1h33_LuceneVarGapDocFreqInterval_0.tib, _1h33_LuceneVarGapDocFreqInterval_0.tiv, _1h34.fdt, _1h34.fdx, _1h34.fnm, _1h34.nvd, _1h34.nvm, _1h34.si, _1h34_LuceneVarGapDocFreqInterval_0.doc, _1h34_LuceneVarGapDocFreqInterval_0.tib, _1h34_LuceneVarGapDocFreqInterval_0.tiv, _1h35.fdt, _1h35.fdx, _1h35.fnm, _1h35.nvd, _1h35.nvm, _1h35.si, _1h35_LuceneVarGapDocFreqInterval_0.doc, _1h35_LuceneVarGapDocFreqInterval_0.tib, _1h35_LuceneVarGapDocFreqInterval_0.tiv, _1h37.fdt, _1h37.fdx, _1h37.fnm, _1h37.nvd, _1h37.nvm, _1h37.si, _1h37_LuceneVarGapDocFreqInterval_0.doc, _1h37_LuceneVarGapDocFreqInterval_0.tib, _1h37_LuceneVarGapDocFreqInterval_0.tiv, _1h38.fdt, _1h38.fdx, _1h38.fnm, _1h38.nvd, _1h38.nvm, _1h38.si, _1h38_LuceneVarGapDocFreqInterval_0.doc, _1h38_LuceneVarGapDocFreqInterval_0.tib, _1h38_LuceneVarGapDocFreqInterval_0.tiv, _1h39.fdt, _1h39.fdx, _1h39.fnm, _1h39.nvd, _1h39.nvm, _1h39.si, _1h39_LuceneVarGapDocFreqInterval_0.doc, _1h39_LuceneVarGapDocFreqInterval_0.tib, _1h39_LuceneVarGapDocFreqInterval_0.tiv, _1h3a.fdt, _1h3a.fdx, _1h3a.fnm, _1h3a.nvd, _1h3a.nvm, _1h3a.si, _1h3a_LuceneVarGapDocFreqInterval_0.doc, _1h3a_LuceneVarGapDocFreqInterval_0.tib, _1h3a_LuceneVarGapDocFreqInterval_0.tiv, _1h3b.fdt, _1h3b.fdx, _1h3b.fnm, _1h3b.nvd, _1h3b.nvm, _1h3b.si, _1h3b_LuceneVarGapDocFreqInterval_0.doc, _1h3b_LuceneVarGapDocFreqInterval_0.tib, _1h3b_LuceneVarGapDocFreqInterval_0.tiv, _1h3c.fdt, _1h3c.fdx, _1h3c.fnm, _1h3c.nvd, _1h3c.nvm, _1h3c.si, _1h3c_LuceneVarGapDocFreqInterval_0.doc, _1h3c_LuceneVarGapDocFreqInterval_0.tib, _1h3c_LuceneVarGapDocFreqInterval_0.tiv, _1h3d.fdt, _1h3d.fdx, _1h3d.fnm, _1h3d.nvd, _1h3d.nvm, _1h3d.si, _1h3d_LuceneVarGapDocFreqInterval_0.doc, _1h3d_LuceneVarGapDocFreqInterval_0.tib, _1h3d_LuceneVarGapDocFreqInterval_0.tiv, _1h3e.fdt, _1h3e.fdx, _1h3e.fnm, _1h3e.nvd, _1h3e.nvm, _1h3e.si, _1h3e_LuceneVarGapDocFreqInterval_0.doc, _1h3e_LuceneVarGapDocFreqInterval_0.tib, _1h3e_LuceneVarGapDocFreqInterval_0.tiv, _1h3f.fdt, _1h3f.fdx, _1h3f.fnm, _1h3f.nvd, _1h3f.nvm, _1h3f.si, _1h3f_LuceneVarGapDocFreqInterval_0.doc, _1h3f_LuceneVarGapDocFreqInterval_0.tib, _1h3f_LuceneVarGapDocFreqInterval_0.tiv, _1h3g.fdt, _1h3g.fdx, _1h3g.fnm, _1h3g.nvd, _1h3g.nvm, _1h3g.si, _1h3g_LuceneVarGapDocFreqInterval_0.doc, _1h3g_LuceneVarGapDocFreqInterval_0.tib, _1h3g_LuceneVarGapDocFreqInterval_0.tiv, _1h3h.fdt, _1h3h.fdx, _1h3h.fnm, _1h3h.nvd, _1h3h.nvm, _1h3h.si, _1h3h_LuceneVarGapDocFreqInterval_0.doc, _1h3h_LuceneVarGapDocFreqInterval_0.tib, _1h3h_LuceneVarGapDocFreqInterval_0.tiv, _1h3i.fdt, _1h3i.fdx, _1h3i.fnm, _1h3i.nvd, _1h3i.nvm, _1h3i.si, _1h3i_LuceneVarGapDocFreqInterval_0.doc, _1h3i_LuceneVarGapDocFreqInterval_0.tib, _1h3i_LuceneVarGapDocFreqInterval_0.tiv, _1h3j.fdt, _1h3j.fdx, _1h3j.fnm, _1h3j.nvd, _1h3j.nvm, _1h3j.si, _1h3j_LuceneVarGapDocFreqInterval_0.doc, _1h3j_LuceneVarGapDocFreqInterval_0.tib, _1h3j_LuceneVarGapDocFreqInterval_0.tiv, _1h3k.fdt, _1h3k.fdx, _1h3k.fnm, _1h3k.nvd, _1h3k.nvm, _1h3k.si, _1h3k_LuceneVarGapDocFreqInterval_0.doc, _1h3k_LuceneVarGapDocFreqInterval_0.tib, _1h3k_LuceneVarGapDocFreqInterval_0.tiv, _su9.fdt, _su9.fdx, _su9.fnm, _su9.nvd, _su9.nvm, _su9.si, _su9_LuceneVarGapDocFreqInterval_0.doc, _su9_LuceneVarGapDocFreqInterval_0.tib, _su9_LuceneVarGapDocFreqInterval_0.tiv, segments_2]}]> but was:<[{indexVersion=1425569204107,generation=3,filelist=[_1h35.fdt, _1h35.fdx, _1h35.fnm, _1h35.nvd, _1h35.nvm, _1h35.si, _1h35_LuceneVarGapDocFreqInterval_0.doc, _1h35_LuceneVarGapDocFreqInterval_0.tib, _1h35_LuceneVarGapDocFreqInterval_0.tiv, _1h36.fdt, _1h36.fdx, _1h36.fnm, _1h36.nvd, _1h36.nvm, _1h36.si, _1h36_LuceneVarGapDocFreqInterval_0.doc, _1h36_LuceneVarGapDocFreqInterval_0.tib, _1h36_LuceneVarGapDocFreqInterval_0.tiv, _1h37.fdt, _1h37.fdx, _1h37.fnm, _1h37.nvd, _1h37.nvm, _1h37.si, _1h37_LuceneVarGapDocFreqInterval_0.doc, _1h37_LuceneVarGapDocFreqInterval_0.tib, _1h37_LuceneVarGapDocFreqInterval_0.tiv, _1h38.fdt, _1h38.fdx, _1h38.fnm, _1h38.nvd, _1h38.nvm, _1h38.si, _1h38_LuceneVarGapDocFreqInterval_0.doc, _1h38_LuceneVarGapDocFreqInterval_0.tib, _1h38_LuceneVarGapDocFreqInterval_0.tiv, _1h39.fdt, _1h39.fdx, _1h39.fnm, _1h39.nvd, _1h39.nvm, _1h39.si, _1h39_LuceneVarGapDocFreqInterval_0.doc, _1h39_LuceneVarGapDocFreqInterval_0.tib, _1h39_LuceneVarGapDocFreqInterval_0.tiv, _1h3a.fdt, _1h3a.fdx, _1h3a.fnm, _1h3a.nvd, _1h3a.nvm, _1h3a.si, _1h3a_LuceneVarGapDocFreqInterval_0.doc, _1h3a_LuceneVarGapDocFreqInterval_0.tib, _1h3a_LuceneVarGapDocFreqInterval_0.tiv, _1h3b.fdt, _1h3b.fdx, _1h3b.fnm, _1h3b.nvd, _1h3b.nvm, _1h3b.si, _1h3b_LuceneVarGapDocFreqInterval_0.doc, _1h3b_LuceneVarGapDocFreqInterval_0.tib, _1h3b_LuceneVarGapDocFreqInterval_0.tiv, _1h3c.fdt, _1h3c.fdx, _1h3c.fnm, _1h3c.nvd, _1h3c.nvm, _1h3c.si, _1h3c_LuceneVarGapDocFreqInterval_0.doc, _1h3c_LuceneVarGapDocFreqInterval_0.tib, _1h3c_LuceneVarGapDocFreqInterval_0.tiv, _1h3d.fdt, _1h3d.fdx, _1h3d.fnm, _1h3d.nvd, _1h3d.nvm, _1h3d.si, _1h3d_LuceneVarGapDocFreqInterval_0.doc, _1h3d_LuceneVarGapDocFreqInterval_0.tib, _1h3d_LuceneVarGapDocFreqInterval_0.tiv, _1h3e.fdt, _1h3e.fdx, _1h3e.fnm, _1h3e.nvd, _1h3e.nvm, _1h3e.si, _1h3e_LuceneVarGapDocFreqInterval_0.doc, _1h3e_LuceneVarGapDocFreqInterval_0.tib, _1h3e_LuceneVarGapDocFreqInterval_0.tiv, _1h3f.fdt, _1h3f.fdx, _1h3f.fnm, _1h3f.nvd, _1h3f.nvm, _1h3f.si, _1h3f_LuceneVarGapDocFreqInterval_0.doc, _1h3f_LuceneVarGapDocFreqInterval_0.tib, _1h3f_LuceneVarGapDocFreqInterval_0.tiv, _1h3g.fdt, _1h3g.fdx, _1h3g.fnm, _1h3g.nvd, _1h3g.nvm, _1h3g.si, _1h3g_LuceneVarGapDocFreqInterval_0.doc, _1h3g_LuceneVarGapDocFreqInterval_0.tib, _1h3g_LuceneVarGapDocFreqInterval_0.tiv, _1h3h.fdt, _1h3h.fdx, _1h3h.fnm, _1h3h.nvd, _1h3h.nvm, _1h3h.si, _1h3h_LuceneVarGapDocFreqInterval_0.doc, _1h3h_LuceneVarGapDocFreqInterval_0.tib, _1h3h_LuceneVarGapDocFreqInterval_0.tiv, _1h3i.fdt, _1h3i.fdx, _1h3i.fnm, _1h3i.nvd, _1h3i.nvm, _1h3i.si, _1h3i_LuceneVarGapDocFreqInterval_0.doc, _1h3i_LuceneVarGapDocFreqInterval_0.tib, _1h3i_LuceneVarGapDocFreqInterval_0.tiv, _1h3j.fdt, _1h3j.fdx, _1h3j.fnm, _1h3j.nvd, _1h3j.nvm, _1h3j.si, _1h3j_LuceneVarGapDocFreqInterval_0.doc, _1h3j_LuceneVarGapDocFreqInterval_0.tib, _1h3j_LuceneVarGapDocFreqInterval_0.tiv, _1h3k.fdt, _1h3k.fdx, _1h3k.fnm, _1h3k.nvd, _1h3k.nvm, _1h3k.si, _1h3k_LuceneVarGapDocFreqInterval_0.doc, _1h3k_LuceneVarGapDocFreqInterval_0.tib, _1h3k_LuceneVarGapDocFreqInterval_0.tiv, _su9.fdt, _su9.fdx, _su9.fnm, _su9.nvd, _su9.nvm, _su9.si, _su9_LuceneVarGapDocFreqInterval_0.doc, _su9_LuceneVarGapDocFreqInterval_0.tib, _su9_LuceneVarGapDocFreqInterval_0.tiv, segments_3]}, {indexVersion=1425569204107,generation=2,filelist=[_1h0i.fdt, _1h0i.fdx, _1h0i.fnm, _1h0i.nvd, _1h0i.nvm, _1h0i.si, _1h0i_LuceneVarGapDocFreqInterval_0.doc, _1h0i_LuceneVarGapDocFreqInterval_0.tib, _1h0i_LuceneVarGapDocFreqInterval_0.tiv, _1h2a.fdt, _1h2a.fdx, _1h2a.fnm, _1h2a.nvd, _1h2a.nvm, _1h2a.si, _1h2a_LuceneVarGapDocFreqInterval_0.doc, _1h2a_LuceneVarGapDocFreqInterval_0.tib, _1h2a_LuceneVarGapDocFreqInterval_0.tiv, _1h2b.cfe, _1h2b.cfs, _1h2b.si, _1h2c.fdt, _1h2c.fdx, _1h2c.fnm, _1h2c.nvd, _1h2c.nvm, _1h2c.si, _1h2c_LuceneVarGapDocFreqInterval_0.doc, _1h2c_LuceneVarGapDocFreqInterval_0.tib, _1h2c_LuceneVarGapDocFreqInterval_0.tiv, _1h2d.fdt, _1h2d.fdx, _1h2d.fnm, _1h2d.nvd, _1h2d.nvm, _1h2d.si, _1h2d_LuceneVarGapDocFreqInterval_0.doc, _1h2d_LuceneVarGapDocFreqInterval_0.tib, _1h2d_LuceneVarGapDocFreqInterval_0.tiv, _1h2e.fdt, _1h2e.fdx, _1h2e.fnm, _1h2e.nvd, _1h2e.nvm, _1h2e.si, _1h2e_LuceneVarGapDocFreqInterval_0.doc, _1h2e_LuceneVarGapDocFreqInterval_0.tib, _1h2e_LuceneVarGapDocFreqInterval_0.tiv, _1h2f.fdt, _1h2f.fdx, _1h2f.fnm, _1h2f.nvd, _1h2f.nvm, _1h2f.si, _1h2f_LuceneVarGapDocFreqInterval_0.doc, _1h2f_LuceneVarGapDocFreqInterval_0.tib, _1h2f_LuceneVarGapDocFreqInterval_0.tiv, _1h2g.fdt, _1h2g.fdx, _1h2g.fnm, _1h2g.nvd, _1h2g.nvm, _1h2g.si, _1h2g_LuceneVarGapDocFreqInterval_0.doc, _1h2g_LuceneVarGapDocFreqInterval_0.tib, _1h2g_LuceneVarGapDocFreqInterval_0.tiv, _1h2h.fdt, _1h2h.fdx, _1h2h.fnm, _1h2h.nvd, _1h2h.nvm, _1h2h.si, _1h2h_LuceneVarGapDocFreqInterval_0.doc, _1h2h_LuceneVarGapDocFreqInterval_0.tib, _1h2h_LuceneVarGapDocFreqInterval_0.tiv, _1h2i.fdt, _1h2i.fdx, _1h2i.fnm, _1h2i.nvd, _1h2i.nvm, _1h2i.si, _1h2i_LuceneVarGapDocFreqInterval_0.doc, _1h2i_LuceneVarGapDocFreqInterval_0.tib, _1h2i_LuceneVarGapDocFreqInterval_0.tiv, _1h2j.fdt, _1h2j.fdx, _1h2j.fnm, _1h2j.nvd, _1h2j.nvm, _1h2j.si, _1h2j_LuceneVarGapDocFreqInterval_0.doc, _1h2j_LuceneVarGapDocFreqInterval_0.tib, _1h2j_LuceneVarGapDocFreqInterval_0.tiv, _1h2k.fdt, _1h2k.fdx, _1h2k.fnm, _1h2k.nvd, _1h2k.nvm, _1h2k.si, _1h2k_LuceneVarGapDocFreqInterval_0.doc, _1h2k_LuceneVarGapDocFreqInterval_0.tib, _1h2k_LuceneVarGapDocFreqInterval_0.tiv, _1h2l.fdt, _1h2l.fdx, _1h2l.fnm, _1h2l.nvd, _1h2l.nvm, _1h2l.si, _1h2l_LuceneVarGapDocFreqInterval_0.doc, _1h2l_LuceneVarGapDocFreqInterval_0.tib, _1h2l_LuceneVarGapDocFreqInterval_0.tiv, _1h2m.fdt, _1h2m.fdx, _1h2m.fnm, _1h2m.nvd, _1h2m.nvm, _1h2m.si, _1h2m_LuceneVarGapDocFreqInterval_0.doc, _1h2m_LuceneVarGapDocFreqInterval_0.tib, _1h2m_LuceneVarGapDocFreqInterval_0.tiv, _1h2n.fdt, _1h2n.fdx, _1h2n.fnm, _1h2n.nvd, _1h2n.nvm, _1h2n.si, _1h2n_LuceneVarGapDocFreqInterval_0.doc, _1h2n_LuceneVarGapDocFreqInterval_0.tib, _1h2n_LuceneVarGapDocFreqInterval_0.tiv, _1h2o.fdt, _1h2o.fdx, _1h2o.fnm, _1h2o.nvd, _1h2o.nvm, _1h2o.si, _1h2o_LuceneVarGapDocFreqInterval_0.doc, _1h2o_LuceneVarGapDocFreqInterval_0.tib, _1h2o_LuceneVarGapDocFreqInterval_0.tiv, _1h2p.fdt, _1h2p.fdx, _1h2p.fnm, _1h2p.nvd, _1h2p.nvm, _1h2p.si, _1h2p_LuceneVarGapDocFreqInterval_0.doc, _1h2p_LuceneVarGapDocFreqInterval_0.tib, _1h2p_LuceneVarGapDocFreqInterval_0.tiv, _1h2q.fdt, _1h2q.fdx, _1h2q.fnm, _1h2q.nvd, _1h2q.nvm, _1h2q.si, _1h2q_LuceneVarGapDocFreqInterval_0.doc, _1h2q_LuceneVarGapDocFreqInterval_0.tib, _1h2q_LuceneVarGapDocFreqInterval_0.tiv, _1h2r.fdt, _1h2r.fdx, _1h2r.fnm, _1h2r.nvd, _1h2r.nvm, _1h2r.si, _1h2r_LuceneVarGapDocFreqInterval_0.doc, _1h2r_LuceneVarGapDocFreqInterval_0.tib, _1h2r_LuceneVarGapDocFreqInterval_0.tiv, _1h2s.fdt, _1h2s.fdx, _1h2s.fnm, _1h2s.nvd, _1h2s.nvm, _1h2s.si, _1h2s_LuceneVarGapDocFreqInterval_0.doc, _1h2s_LuceneVarGapDocFreqInterval_0.tib, _1h2s_LuceneVarGapDocFreqInterval_0.tiv, _1h2t.fdt, _1h2t.fdx, _1h2t.fnm, _1h2t.nvd, _1h2t.nvm, _1h2t.si, _1h2t_LuceneVarGapDocFreqInterval_0.doc, _1h2t_LuceneVarGapDocFreqInterval_0.tib, _1h2t_LuceneVarGapDocFreqInterval_0.tiv, _1h2u.fdt, _1h2u.fdx, _1h2u.fnm, _1h2u.nvd, _1h2u.nvm, _1h2u.si, _1h2u_LuceneVarGapDocFreqInterval_0.doc, _1h2u_LuceneVarGapDocFreqInterval_0.tib, _1h2u_LuceneVarGapDocFreqInterval_0.tiv, _1h2v.fdt, _1h2v.fdx, _1h2v.fnm, _1h2v.nvd, _1h2v.nvm, _1h2v.si, _1h2v_LuceneVarGapDocFreqInterval_0.doc, _1h2v_LuceneVarGapDocFreqInterval_0.tib, _1h2v_LuceneVarGapDocFreqInterval_0.tiv, _1h2w.fdt, _1h2w.fdx, _1h2w.fnm, _1h2w.nvd, _1h2w.nvm, _1h2w.si, _1h2w_LuceneVarGapDocFreqInterval_0.doc, _1h2w_LuceneVarGapDocFreqInterval_0.tib, _1h2w_LuceneVarGapDocFreqInterval_0.tiv, _1h2x.fdt, _1h2x.fdx, _1h2x.fnm, _1h2x.nvd, _1h2x.nvm, _1h2x.si, _1h2x_LuceneVarGapDocFreqInterval_0.doc, _1h2x_LuceneVarGapDocFreqInterval_0.tib, _1h2x_LuceneVarGapDocFreqInterval_0.tiv, _1h2y.fdt, _1h2y.fdx, _1h2y.fnm, _1h2y.nvd, _1h2y.nvm, _1h2y.si, _1h2y_LuceneVarGapDocFreqInterval_0.doc, _1h2y_LuceneVarGapDocFreqInterval_0.tib, _1h2y_LuceneVarGapDocFreqInterval_0.tiv, _1h2z.fdt, _1h2z.fdx, _1h2z.fnm, _1h2z.nvd, _1h2z.nvm, _1h2z.si, _1h2z_LuceneVarGapDocFreqInterval_0.doc, _1h2z_LuceneVarGapDocFreqInterval_0.tib, _1h2z_LuceneVarGapDocFreqInterval_0.tiv, _1h30.fdt, _1h30.fdx, _1h30.fnm, _1h30.nvd, _1h30.nvm, _1h30.si, _1h30_LuceneVarGapDocFreqInterval_0.doc, _1h30_LuceneVarGapDocFreqInterval_0.tib, _1h30_LuceneVarGapDocFreqInterval_0.tiv, _1h31.fdt, _1h31.fdx, _1h31.fnm, _1h31.nvd, _1h31.nvm, _1h31.si, _1h31_LuceneVarGapDocFreqInterval_0.doc, _1h31_LuceneVarGapDocFreqInterval_0.tib, _1h31_LuceneVarGapDocFreqInterval_0.tiv, _1h32.fdt, _1h32.fdx, _1h32.fnm, _1h32.nvd, _1h32.nvm, _1h32.si, _1h32_LuceneVarGapDocFreqInterval_0.doc, _1h32_LuceneVarGapDocFreqInterval_0.tib, _1h32_LuceneVarGapDocFreqInterval_0.tiv, _1h33.fdt, _1h33.fdx, _1h33.fnm, _1h33.nvd, _1h33.nvm, _1h33.si, _1h33_LuceneVarGapDocFreqInterval_0.doc, _1h33_LuceneVarGapDocFreqInterval_0.tib, _1h33_LuceneVarGapDocFreqInterval_0.tiv, _1h34.fdt, _1h34.fdx, _1h34.fnm, _1h34.nvd, _1h34.nvm, _1h34.si, _1h34_LuceneVarGapDocFreqInterval_0.doc, _1h34_LuceneVarGapDocFreqInterval_0.tib, _1h34_LuceneVarGapDocFreqInterval_0.tiv, _1h35.fdt, _1h35.fdx, _1h35.fnm, _1h35.nvd, _1h35.nvm, _1h35.si, _1h35_LuceneVarGapDocFreqInterval_0.doc, _1h35_LuceneVarGapDocFreqInterval_0.tib, _1h35_LuceneVarGapDocFreqInterval_0.tiv, _1h37.fdt, _1h37.fdx, _1h37.fnm, _1h37.nvd, _1h37.nvm, _1h37.si, _1h37_LuceneVarGapDocFreqInterval_0.doc, _1h37_LuceneVarGapDocFreqInterval_0.tib, _1h37_LuceneVarGapDocFreqInterval_0.tiv, _1h38.fdt, _1h38.fdx, _1h38.fnm, _1h38.nvd, _1h38.nvm, _1h38.si, _1h38_LuceneVarGapDocFreqInterval_0.doc, _1h38_LuceneVarGapDocFreqInterval_0.tib, _1h38_LuceneVarGapDocFreqInterval_0.tiv, _1h39.fdt, _1h39.fdx, _1h39.fnm, _1h39.nvd, _1h39.nvm, _1h39.si, _1h39_LuceneVarGapDocFreqInterval_0.doc, _1h39_LuceneVarGapDocFreqInterval_0.tib, _1h39_LuceneVarGapDocFreqInterval_0.tiv, _1h3a.fdt, _1h3a.fdx, _1h3a.fnm, _1h3a.nvd, _1h3a.nvm, _1h3a.si, _1h3a_LuceneVarGapDocFreqInterval_0.doc, _1h3a_LuceneVarGapDocFreqInterval_0.tib, _1h3a_LuceneVarGapDocFreqInterval_0.tiv, _1h3b.fdt, _1h3b.fdx, _1h3b.fnm, _1h3b.nvd, _1h3b.nvm, _1h3b.si, _1h3b_LuceneVarGapDocFreqInterval_0.doc, _1h3b_LuceneVarGapDocFreqInterval_0.tib, _1h3b_LuceneVarGapDocFreqInterval_0.tiv, _1h3c.fdt, _1h3c.fdx, _1h3c.fnm, _1h3c.nvd, _1h3c.nvm, _1h3c.si, _1h3c_LuceneVarGapDocFreqInterval_0.doc, _1h3c_LuceneVarGapDocFreqInterval_0.tib, _1h3c_LuceneVarGapDocFreqInterval_0.tiv, _1h3d.fdt, _1h3d.fdx, _1h3d.fnm, _1h3d.nvd, _1h3d.nvm, _1h3d.si, _1h3d_LuceneVarGapDocFreqInterval_0.doc, _1h3d_LuceneVarGapDocFreqInterval_0.tib, _1h3d_LuceneVarGapDocFreqInterval_0.tiv, _1h3e.fdt, _1h3e.fdx, _1h3e.fnm, _1h3e.nvd, _1h3e.nvm, _1h3e.si, _1h3e_LuceneVarGapDocFreqInterval_0.doc, _1h3e_LuceneVarGapDocFreqInterval_0.tib, _1h3e_LuceneVarGapDocFreqInterval_0.tiv, _1h3f.fdt, _1h3f.fdx, _1h3f.fnm, _1h3f.nvd, _1h3f.nvm, _1h3f.si, _1h3f_LuceneVarGapDocFreqInterval_0.doc, _1h3f_LuceneVarGapDocFreqInterval_0.tib, _1h3f_LuceneVarGapDocFreqInterval_0.tiv, _1h3g.fdt, _1h3g.fdx, _1h3g.fnm, _1h3g.nvd, _1h3g.nvm, _1h3g.si, _1h3g_LuceneVarGapDocFreqInterval_0.doc, _1h3g_LuceneVarGapDocFreqInterval_0.tib, _1h3g_LuceneVarGapDocFreqInterval_0.tiv, _1h3h.fdt, _1h3h.fdx, _1h3h.fnm, _1h3h.nvd, _1h3h.nvm, _1h3h.si, _1h3h_LuceneVarGapDocFreqInterval_0.doc, _1h3h_LuceneVarGapDocFreqInterval_0.tib, _1h3h_LuceneVarGapDocFreqInterval_0.tiv, _1h3i.fdt, _1h3i.fdx, _1h3i.fnm, _1h3i.nvd, _1h3i.nvm, _1h3i.si, _1h3i_LuceneVarGapDocFreqInterval_0.doc, _1h3i_LuceneVarGapDocFreqInterval_0.tib, _1h3i_LuceneVarGapDocFreqInterval_0.tiv, _1h3j.fdt, _1h3j.fdx, _1h3j.fnm, _1h3j.nvd, _1h3j.nvm, _1h3j.si, _1h3j_LuceneVarGapDocFreqInterval_0.doc, _1h3j_LuceneVarGapDocFreqInterval_0.tib, _1h3j_LuceneVarGapDocFreqInterval_0.tiv, _1h3k.fdt, _1h3k.fdx, _1h3k.fnm, _1h3k.nvd, _1h3k.nvm, _1h3k.si, _1h3k_LuceneVarGapDocFreqInterval_0.doc, _1h3k_LuceneVarGapDocFreqInterval_0.tib, _1h3k_LuceneVarGapDocFreqInterval_0.tiv, _su9.fdt, _su9.fdx, _su9.fnm, _su9.nvd, _su9.nvm, _su9.si, _su9_LuceneVarGapDocFreqInterval_0.doc, _su9_LuceneVarGapDocFreqInterval_0.tib, _su9_LuceneVarGapDocFreqInterval_0.tiv, segments_2]}]>

Stack Trace:
java.lang.AssertionError: expected:<[{indexVersion=1425569204107,generation=2,filelist=[_1h0i.fdt, _1h0i.fdx, _1h0i.fnm, _1h0i.nvd, _1h0i.nvm, _1h0i.si, _1h0i_LuceneVarGapDocFreqInterval_0.doc, _1h0i_LuceneVarGapDocFreqInterval_0.tib, _1h0i_LuceneVarGapDocFreqInterval_0.tiv, _1h2a.fdt, _1h2a.fdx, _1h2a.fnm, _1h2a.nvd, _1h2a.nvm, _1h2a.si, _1h2a_LuceneVarGapDocFreqInterval_0.doc, _1h2a_LuceneVarGapDocFreqInterval_0.tib, _1h2a_LuceneVarGapDocFreqInterval_0.tiv, _1h2b.cfe, _1h2b.cfs, _1h2b.si, _1h2c.fdt, _1h2c.fdx, _1h2c.fnm, _1h2c.nvd, _1h2c.nvm, _1h2c.si, _1h2c_LuceneVarGapDocFreqInterval_0.doc, _1h2c_LuceneVarGapDocFreqInterval_0.tib, _1h2c_LuceneVarGapDocFreqInterval_0.tiv, _1h2d.fdt, _1h2d.fdx, _1h2d.fnm, _1h2d.nvd, _1h2d.nvm, _1h2d.si, _1h2d_LuceneVarGapDocFreqInterval_0.doc, _1h2d_LuceneVarGapDocFreqInterval_0.tib, _1h2d_LuceneVarGapDocFreqInterval_0.tiv, _1h2e.fdt, _1h2e.fdx, _1h2e.fnm, _1h2e.nvd, _1h2e.nvm, _1h2e.si, _1h2e_LuceneVarGapDocFreqInterval_0.doc, _1h2e_LuceneVarGapDocFreqInterval_0.tib, _1h2e_LuceneVarGapDocFreqInterval_0.tiv, _1h2f.fdt, _1h2f.fdx, _1h2f.fnm, _1h2f.nvd, _1h2f.nvm, _1h2f.si, _1h2f_LuceneVarGapDocFreqInterval_0.doc, _1h2f_LuceneVarGapDocFreqInterval_0.tib, _1h2f_LuceneVarGapDocFreqInterval_0.tiv, _1h2g.fdt, _1h2g.fdx, _1h2g.fnm, _1h2g.nvd, _1h2g.nvm, _1h2g.si, _1h2g_LuceneVarGapDocFreqInterval_0.doc, _1h2g_LuceneVarGapDocFreqInterval_0.tib, _1h2g_LuceneVarGapDocFreqInterval_0.tiv, _1h2h.fdt, _1h2h.fdx, _1h2h.fnm, _1h2h.nvd, _1h2h.nvm, _1h2h.si, _1h2h_LuceneVarGapDocFreqInterval_0.doc, _1h2h_LuceneVarGapDocFreqInterval_0.tib, _1h2h_LuceneVarGapDocFreqInterval_0.tiv, _1h2i.fdt, _1h2i.fdx, _1h2i.fnm, _1h2i.nvd, _1h2i.nvm, _1h2i.si, _1h2i_LuceneVarGapDocFreqInterval_0.doc, _1h2i_LuceneVarGapDocFreqInterval_0.tib, _1h2i_LuceneVarGapDocFreqInterval_0.tiv, _1h2j.fdt, _1h2j.fdx, _1h2j.fnm, _1h2j.nvd, _1h2j.nvm, _1h2j.si, _1h2j_LuceneVarGapDocFreqInterval_0.doc, _1h2j_LuceneVarGapDocFreqInterval_0.tib, _1h2j_LuceneVarGapDocFreqInterval_0.tiv, _1h2k.fdt, _1h2k.fdx, _1h2k.fnm, _1h2k.nvd, _1h2k.nvm, _1h2k.si, _1h2k_LuceneVarGapDocFreqInterval_0.doc, _1h2k_LuceneVarGapDocFreqInterval_0.tib, _1h2k_LuceneVarGapDocFreqInterval_0.tiv, _1h2l.fdt, _1h2l.fdx, _1h2l.fnm, _1h2l.nvd, _1h2l.nvm, _1h2l.si, _1h2l_LuceneVarGapDocFreqInterval_0.doc, _1h2l_LuceneVarGapDocFreqInterval_0.tib, _1h2l_LuceneVarGapDocFreqInterval_0.tiv, _1h2m.fdt, _1h2m.fdx, _1h2m.fnm, _1h2m.nvd, _1h2m.nvm, _1h2m.si, _1h2m_LuceneVarGapDocFreqInterval_0.doc, _1h2m_LuceneVarGapDocFreqInterval_0.tib, _1h2m_LuceneVarGapDocFreqInterval_0.tiv, _1h2n.fdt, _1h2n.fdx, _1h2n.fnm, _1h2n.nvd, _1h2n.nvm, _1h2n.si, _1h2n_LuceneVarGapDocFreqInterval_0.doc, _1h2n_LuceneVarGapDocFreqInterval_0.tib, _1h2n_LuceneVarGapDocFreqInterval_0.tiv, _1h2o.fdt, _1h2o.fdx, _1h2o.fnm, _1h2o.nvd, _1h2o.nvm, _1h2o.si, _1h2o_LuceneVarGapDocFreqInterval_0.doc, _1h2o_LuceneVarGapDocFreqInterval_0.tib, _1h2o_LuceneVarGapDocFreqInterval_0.tiv, _1h2p.fdt, _1h2p.fdx, _1h2p.fnm, _1h2p.nvd, _1h2p.nvm, _1h2p.si, _1h2p_LuceneVarGapDocFreqInterval_0.doc, _1h2p_LuceneVarGapDocFreqInterval_0.tib, _1h2p_LuceneVarGapDocFreqInterval_0.tiv, _1h2q.fdt, _1h2q.fdx, _1h2q.fnm, _1h2q.nvd, _1h2q.nvm, _1h2q.si, _1h2q_LuceneVarGapDocFreqInterval_0.doc, _1h2q_LuceneVarGapDocFreqInterval_0.tib, _1h2q_LuceneVarGapDocFreqInterval_0.tiv, _1h2r.fdt, _1h2r.fdx, _1h2r.fnm, _1h2r.nvd, _1h2r.nvm, _1h2r.si, _1h2r_LuceneVarGapDocFreqInterval_0.doc, _1h2r_LuceneVarGapDocFreqInterval_0.tib, _1h2r_LuceneVarGapDocFreqInterval_0.tiv, _1h2s.fdt, _1h2s.fdx, _1h2s.fnm, _1h2s.nvd, _1h2s.nvm, _1h2s.si, _1h2s_LuceneVarGapDocFreqInterval_0.doc, _1h2s_LuceneVarGapDocFreqInterval_0.tib, _1h2s_LuceneVarGapDocFreqInterval_0.tiv, _1h2t.fdt, _1h2t.fdx, _1h2t.fnm, _1h2t.nvd, _1h2t.nvm, _1h2t.si, _1h2t_LuceneVarGapDocFreqInterval_0.doc, _1h2t_LuceneVarGapDocFreqInterval_0.tib, _1h2t_LuceneVarGapDocFreqInterval_0.tiv, _1h2u.fdt, _1h2u.fdx, _1h2u.fnm, _1h2u.nvd, _1h2u.nvm, _1h2u.si, _1h2u_LuceneVarGapDocFreqInterval_0.doc, _1h2u_LuceneVarGapDocFreqInterval_0.tib, _1h2u_LuceneVarGapDocFreqInterval_0.tiv, _1h2v.fdt, _1h2v.fdx, _1h2v.fnm, _1h2v.nvd, _1h2v.nvm, _1h2v.si, _1h2v_LuceneVarGapDocFreqInterval_0.doc, _1h2v_LuceneVarGapDocFreqInterval_0.tib, _1h2v_LuceneVarGapDocFreqInterval_0.tiv, _1h2w.fdt, _1h2w.fdx, _1h2w.fnm, _1h2w.nvd, _1h2w.nvm, _1h2w.si, _1h2w_LuceneVarGapDocFreqInterval_0.doc, _1h2w_LuceneVarGapDocFreqInterval_0.tib, _1h2w_LuceneVarGapDocFreqInterval_0.tiv, _1h2x.fdt, _1h2x.fdx, _1h2x.fnm, _1h2x.nvd, _1h2x.nvm, _1h2x.si, _1h2x_LuceneVarGapDocFreqInterval_0.doc, _1h2x_LuceneVarGapDocFreqInterval_0.tib, _1h2x_LuceneVarGapDocFreqInterval_0.tiv, _1h2y.fdt, _1h2y.fdx, _1h2y.fnm, _1h2y.nvd, _1h2y.nvm, _1h2y.si, _1h2y_LuceneVarGapDocFreqInterval_0.doc, _1h2y_LuceneVarGapDocFreqInterval_0.tib, _1h2y_LuceneVarGapDocFreqInterval_0.tiv, _1h2z.fdt, _1h2z.fdx, _1h2z.fnm, _1h2z.nvd, _1h2z.nvm, _1h2z.si, _1h2z_LuceneVarGapDocFreqInterval_0.doc, _1h2z_LuceneVarGapDocFreqInterval_0.tib, _1h2z_LuceneVarGapDocFreqInterval_0.tiv, _1h30.fdt, _1h30.fdx, _1h30.fnm, _1h30.nvd, _1h30.nvm, _1h30.si, _1h30_LuceneVarGapDocFreqInterval_0.doc, _1h30_LuceneVarGapDocFreqInterval_0.tib, _1h30_LuceneVarGapDocFreqInterval_0.tiv, _1h31.fdt, _1h31.fdx, _1h31.fnm, _1h31.nvd, _1h31.nvm, _1h31.si, _1h31_LuceneVarGapDocFreqInterval_0.doc, _1h31_LuceneVarGapDocFreqInterval_0.tib, _1h31_LuceneVarGapDocFreqInterval_0.tiv, _1h32.fdt, _1h32.fdx, _1h32.fnm, _1h32.nvd, _1h32.nvm, _1h32.si, _1h32_LuceneVarGapDocFreqInterval_0.doc, _1h32_LuceneVarGapDocFreqInterval_0.tib, _1h32_LuceneVarGapDocFreqInterval_0.tiv, _1h33.fdt, _1h33.fdx, _1h33.fnm, _1h33.nvd, _1h33.nvm, _1h33.si, _1h33_LuceneVarGapDocFreqInterval_0.doc, _1h33_LuceneVarGapDocFreqInterval_0.tib, _1h33_LuceneVarGapDocFreqInterval_0.tiv, _1h34.fdt, _1h34.fdx, _1h34.fnm, _1h34.nvd, _1h34.nvm, _1h34.si, _1h34_LuceneVarGapDocFreqInterval_0.doc, _1h34_LuceneVarGapDocFreqInterval_0.tib, _1h34_LuceneVarGapDocFreqInterval_0.tiv, _1h35.fdt, _1h35.fdx, _1h35.fnm, _1h35.nvd, _1h35.nvm, _1h35.si, _1h35_LuceneVarGapDocFreqInterval_0.doc, _1h35_LuceneVarGapDocFreqInterval_0.tib, _1h35_LuceneVarGapDocFreqInterval_0.tiv, _1h37.fdt, _1h37.fdx, _1h37.fnm, _1h37.nvd, _1h37.nvm, _1h37.si, _1h37_LuceneVarGapDocFreqInterval_0.doc, _1h37_LuceneVarGapDocFreqInterval_0.tib, _1h37_LuceneVarGapDocFreqInterval_0.tiv, _1h38.fdt, _1h38.fdx, _1h38.fnm, _1h38.nvd, _1h38.nvm, _1h38.si, _1h38_LuceneVarGapDocFreqInterval_0.doc, _1h38_LuceneVarGapDocFreqInterval_0.tib, _1h38_LuceneVarGapDocFreqInterval_0.tiv, _1h39.fdt, _1h39.fdx, _1h39.fnm, _1h39.nvd, _1h39.nvm, _1h39.si, _1h39_LuceneVarGapDocFreqInterval_0.doc, _1h39_LuceneVarGapDocFreqInterval_0.tib, _1h39_LuceneVarGapDocFreqInterval_0.tiv, _1h3a.fdt, _1h3a.fdx, _1h3a.fnm, _1h3a.nvd, _1h3a.nvm, _1h3a.si, _1h3a_LuceneVarGapDocFreqInterval_0.doc, _1h3a_LuceneVarGapDocFreqInterval_0.tib, _1h3a_LuceneVarGapDocFreqInterval_0.tiv, _1h3b.fdt, _1h3b.fdx, _1h3b.fnm, _1h3b.nvd, _1h3b.nvm, _1h3b.si, _1h3b_LuceneVarGapDocFreqInterval_0.doc, _1h3b_LuceneVarGapDocFreqInterval_0.tib, _1h3b_LuceneVarGapDocFreqInterval_0.tiv, _1h3c.fdt, _1h3c.fdx, _1h3c.fnm, _1h3c.nvd, _1h3c.nvm, _1h3c.si, _1h3c_LuceneVarGapDocFreqInterval_0.doc, _1h3c_LuceneVarGapDocFreqInterval_0.tib, _1h3c_LuceneVarGapDocFreqInterval_0.tiv, _1h3d.fdt, _1h3d.fdx, _1h3d.fnm, _1h3d.nvd, _1h3d.nvm, _1h3d.si, _1h3d_LuceneVarGapDocFreqInterval_0.doc, _1h3d_LuceneVarGapDocFreqInterval_0.tib, _1h3d_LuceneVarGapDocFreqInterval_0.tiv, _1h3e.fdt, _1h3e.fdx, _1h3e.fnm, _1h3e.nvd, _1h3e.nvm, _1h3e.si, _1h3e_LuceneVarGapDocFreqInterval_0.doc, _1h3e_LuceneVarGapDocFreqInterval_0.tib, _1h3e_LuceneVarGapDocFreqInterval_0.tiv, _1h3f.fdt, _1h3f.fdx, _1h3f.fnm, _1h3f.nvd, _1h3f.nvm, _1h3f.si, _1h3f_LuceneVarGapDocFreqInterval_0.doc, _1h3f_LuceneVarGapDocFreqInterval_0.tib, _1h3f_LuceneVarGapDocFreqInterval_0.tiv, _1h3g.fdt, _1h3g.fdx, _1h3g.fnm, _1h3g.nvd, _1h3g.nvm, _1h3g.si, _1h3g_LuceneVarGapDocFreqInterval_0.doc, _1h3g_LuceneVarGapDocFreqInterval_0.tib, _1h3g_LuceneVarGapDocFreqInterval_0.tiv, _1h3h.fdt, _1h3h.fdx, _1h3h.fnm, _1h3h.nvd, _1h3h.nvm, _1h3h.si, _1h3h_LuceneVarGapDocFreqInterval_0.doc, _1h3h_LuceneVarGapDocFreqInterval_0.tib, _1h3h_LuceneVarGapDocFreqInterval_0.tiv, _1h3i.fdt, _1h3i.fdx, _1h3i.fnm, _1h3i.nvd, _1h3i.nvm, _1h3i.si, _1h3i_LuceneVarGapDocFreqInterval_0.doc, _1h3i_LuceneVarGapDocFreqInterval_0.tib, _1h3i_LuceneVarGapDocFreqInterval_0.tiv, _1h3j.fdt, _1h3j.fdx, _1h3j.fnm, _1h3j.nvd, _1h3j.nvm, _1h3j.si, _1h3j_LuceneVarGapDocFreqInterval_0.doc, _1h3j_LuceneVarGapDocFreqInterval_0.tib, _1h3j_LuceneVarGapDocFreqInterval_0.tiv, _1h3k.fdt, _1h3k.fdx, _1h3k.fnm, _1h3k.nvd, _1h3k.nvm, _1h3k.si, _1h3k_LuceneVarGapDocFreqInterval_0.doc, _1h3k_LuceneVarGapDocFreqInterval_0.tib, _1h3k_LuceneVarGapDocFreqInterval_0.tiv, _su9.fdt, _su9.fdx, _su9.fnm, _su9.nvd, _su9.nvm, _su9.si, _su9_LuceneVarGapDocFreqInterval_0.doc, _su9_LuceneVarGapDocFreqInterval_0.tib, _su9_LuceneVarGapDocFreqInterval_0.tiv, segments_2]}]> but was:<[{indexVersion=1425569204107,generation=3,filelist=[_1h35.fdt, _1h35.fdx, _1h35.fnm, _1h35.nvd, _1h35.nvm, _1h35.si, _1h35_LuceneVarGapDocFreqInterval_0.doc, _1h35_LuceneVarGapDocFreqInterval_0.tib, _1h35_LuceneVarGapDocFreqInterval_0.tiv, _1h36.fdt, _1h36.fdx, _1h36.fnm, _1h36.nvd, _1h36.nvm, _1h36.si, _1h36_LuceneVarGapDocFreqInterval_0.doc, _1h36_LuceneVarGapDocFreqInterval_0.tib, _1h36_LuceneVarGapDocFreqInterval_0.tiv, _1h37.fdt, _1h37.fdx, _1h37.fnm, _1h37.nvd, _1h37.nvm, _1h37.si, _1h37_LuceneVarGapDocFreqInterval_0.doc, _1h37_LuceneVarGapDocFreqInterval_0.tib, _1h37_LuceneVarGapDocFreqInterval_0.tiv, _1h38.fdt, _1h38.fdx, _1h38.fnm, _1h38.nvd, _1h38.nvm, _1h38.si, _1h38_LuceneVarGapDocFreqInterval_0.doc, _1h38_LuceneVarGapDocFreqInterval_0.tib, _1h38_LuceneVarGapDocFreqInterval_0.tiv, _1h39.fdt, _1h39.fdx, _1h39.fnm, _1h39.nvd, _1h39.nvm, _1h39.si, _1h39_LuceneVarGapDocFreqInterval_0.doc, _1h39_LuceneVarGapDocFreqInterval_0.tib, _1h39_LuceneVarGapDocFreqInterval_0.tiv, _1h3a.fdt, _1h3a.fdx, _1h3a.fnm, _1h3a.nvd, _1h3a.nvm, _1h3a.si, _1h3a_LuceneVarGapDocFreqInterval_0.doc, _1h3a_LuceneVarGapDocFreqInterval_0.tib, _1h3a_LuceneVarGapDocFreqInterval_0.tiv, _1h3b.fdt, _1h3b.fdx, _1h3b.fnm, _1h3b.nvd, _1h3b.nvm, _1h3b.si, _1h3b_LuceneVarGapDocFreqInterval_0.doc, _1h3b_LuceneVarGapDocFreqInterval_0.tib, _1h3b_LuceneVarGapDocFreqInterval_0.tiv, _1h3c.fdt, _1h3c.fdx, _1h3c.fnm, _1h3c.nvd, _1h3c.nvm, _1h3c.si, _1h3c_LuceneVarGapDocFreqInterval_0.doc, _1h3c_LuceneVarGapDocFreqInterval_0.tib, _1h3c_LuceneVarGapDocFreqInterval_0.tiv, _1h3d.fdt, _1h3d.fdx, _1h3d.fnm, _1h3d.nvd, _1h3d.nvm, _1h3d.si, _1h3d_LuceneVarGapDocFreqInterval_0.doc, _1h3d_LuceneVarGapDocFreqInterval_0.tib, _1h3d_LuceneVarGapDocFreqInterval_0.tiv, _1h3e.fdt, _1h3e.fdx, _1h3e.fnm, _1h3e.nvd, _1h3e.nvm, _1h3e.si, _1h3e_LuceneVarGapDocFreqInterval_0.doc, _1h3e_LuceneVarGapDocFreqInterval_0.tib, _1h3e_LuceneVarGapDocFreqInterval_0.tiv, _1h3f.fdt, _1h3f.fdx, _1h3f.fnm, _1h3f.nvd, _1h3f.nvm, _1h3f.si, _1h3f_LuceneVarGapDocFreqInterval_0.doc, _1h3f_LuceneVarGapDocFreqInterval_0.tib, _1h3f_LuceneVarGapDocFreqInterval_0.tiv, _1h3g.fdt, _1h3g.fdx, _1h3g.fnm, _1h3g.nvd, _1h3g.nvm, _1h3g.si, _1h3g_LuceneVarGapDocFreqInterval_0.doc, _1h3g_LuceneVarGapDocFreqInterval_0.tib, _1h3g_LuceneVarGapDocFreqInterval_0.tiv, _1h3h.fdt, _1h3h.fdx, _1h3h.fnm, _1h3h.nvd, _1h3h.nvm, _1h3h.si, _1h3h_LuceneVarGapDocFreqInterval_0.doc, _1h3h_LuceneVarGapDocFreqInterval_0.tib, _1h3h_LuceneVarGapDocFreqInterval_0.tiv, _1h3i.fdt, _1h3i.fdx, _1h3i.fnm, _1h3i.nvd, _1h3i.nvm, _1h3i.si, _1h3i_LuceneVarGapDocFreqInterval_0.doc, _1h3i_LuceneVarGapDocFreqInterval_0.tib, _1h3i_LuceneVarGapDocFreqInterval_0.tiv, _1h3j.fdt, _1h3j.fdx, _1h3j.fnm, _1h3j.nvd, _1h3j.nvm, _1h3j.si, _1h3j_LuceneVarGapDocFreqInterval_0.doc, _1h3j_LuceneVarGapDocFreqInterval_0.tib, _1h3j_LuceneVarGapDocFreqInterval_0.tiv, _1h3k.fdt, _1h3k.fdx, _1h3k.fnm, _1h3k.nvd, _1h3k.nvm, _1h3k.si, _1h3k_LuceneVarGapDocFreqInterval_0.doc, _1h3k_LuceneVarGapDocFreqInterval_0.tib, _1h3k_LuceneVarGapDocFreqInterval_0.tiv, _su9.fdt, _su9.fdx, _su9.fnm, _su9.nvd, _su9.nvm, _su9.si, _su9_LuceneVarGapDocFreqInterval_0.doc, _su9_LuceneVarGapDocFreqInterval_0.tib, _su9_LuceneVarGapDocFreqInterval_0.tiv, segments_3]}, {indexVersion=1425569204107,generation=2,filelist=[_1h0i.fdt, _1h0i.fdx, _1h0i.fnm, _1h0i.nvd, _1h0i.nvm, _1h0i.si, _1h0i_LuceneVarGapDocFreqInterval_0.doc, _1h0i_LuceneVarGapDocFreqInterval_0.tib, _1h0i_LuceneVarGapDocFreqInterval_0.tiv, _1h2a.fdt, _1h2a.fdx, _1h2a.fnm, _1h2a.nvd, _1h2a.nvm, _1h2a.si, _1h2a_LuceneVarGapDocFreqInterval_0.doc, _1h2a_LuceneVarGapDocFreqInterval_0.tib, _1h2a_LuceneVarGapDocFreqInterval_0.tiv, _1h2b.cfe, _1h2b.cfs, _1h2b.si, _1h2c.fdt, _1h2c.fdx, _1h2c.fnm, _1h2c.nvd, _1h2c.nvm, _1h2c.si, _1h2c_LuceneVarGapDocFreqInterval_0.doc, _1h2c_LuceneVarGapDocFreqInterval_0.tib, _1h2c_LuceneVarGapDocFreqInterval_0.tiv, _1h2d.fdt, _1h2d.fdx, _1h2d.fnm, _1h2d.nvd, _1h2d.nvm, _1h2d.si, _1h2d_LuceneVarGapDocFreqInterval_0.doc, _1h2d_LuceneVarGapDocFreqInterval_0.tib, _1h2d_LuceneVarGapDocFreqInterval_0.tiv, _1h2e.fdt, _1h2e.fdx, _1h2e.fnm, _1h2e.nvd, _1h2e.nvm, _1h2e.si, _1h2e_LuceneVarGapDocFreqInterval_0.doc, _1h2e_LuceneVarGapDocFreqInterval_0.tib, _1h2e_LuceneVarGapDocFreqInterval_0.tiv, _1h2f.fdt, _1h2f.fdx, _1h2f.fnm, _1h2f.nvd, _1h2f.nvm, _1h2f.si, _1h2f_LuceneVarGapDocFreqInterval_0.doc, _1h2f_LuceneVarGapDocFreqInterval_0.tib, _1h2f_LuceneVarGapDocFreqInterval_0.tiv, _1h2g.fdt, _1h2g.fdx, _1h2g.fnm, _1h2g.nvd, _1h2g.nvm, _1h2g.si, _1h2g_LuceneVarGapDocFreqInterval_0.doc, _1h2g_LuceneVarGapDocFreqInterval_0.tib, _1h2g_LuceneVarGapDocFreqInterval_0.tiv, _1h2h.fdt, _1h2h.fdx, _1h2h.fnm, _1h2h.nvd, _1h2h.nvm, _1h2h.si, _1h2h_LuceneVarGapDocFreqInterval_0.doc, _1h2h_LuceneVarGapDocFreqInterval_0.tib, _1h2h_LuceneVarGapDocFreqInterval_0.tiv, _1h2i.fdt, _1h2i.fdx, _1h2i.fnm, _1h2i.nvd, _1h2i.nvm, _1h2i.si, _1h2i_LuceneVarGapDocFreqInterval_0.doc, _1h2i_LuceneVarGapDocFreqInterval_0.tib, _1h2i_LuceneVarGapDocFreqInterval_0.tiv, _1h2j.fdt, _1h2j.fdx, _1h2j.fnm, _1h2j.nvd, _1h2j.nvm, _1h2j.si, _1h2j_LuceneVarGapDocFreqInterval_0.doc, _1h2j_LuceneVarGapDocFreqInterval_0.tib, _1h2j_LuceneVarGapDocFreqInterval_0.tiv, _1h2k.fdt, _1h2k.fdx, _1h2k.fnm, _1h2k.nvd, _1h2k.nvm, _1h2k.si, _1h2k_LuceneVarGapDocFreqInterval_0.doc, _1h2k_LuceneVarGapDocFreqInterval_0.tib, _1h2k_LuceneVarGapDocFreqInterval_0.tiv, _1h2l.fdt, _1h2l.fdx, _1h2l.fnm, _1h2l.nvd, _1h2l.nvm, _1h2l.si, _1h2l_LuceneVarGapDocFreqInterval_0.doc, _1h2l_LuceneVarGapDocFreqInterval_0.tib, _1h2l_LuceneVarGapDocFreqInterval_0.tiv, _1h2m.fdt, _1h2m.fdx, _1h2m.fnm, _1h2m.nvd, _1h2m.nvm, _1h2m.si, _1h2m_LuceneVarGapDocFreqInterval_0.doc, _1h2m_LuceneVarGapDocFreqInterval_0.tib, _1h2m_LuceneVarGapDocFreqInterval_0.tiv, _1h2n.fdt, _1h2n.fdx, _1h2n.fnm, _1h2n.nvd, _1h2n.nvm, _1h2n.si, _1h2n_LuceneVarGapDocFreqInterval_0.doc, _1h2n_LuceneVarGapDocFreqInterval_0.tib, _1h2n_LuceneVarGapDocFreqInterval_0.tiv, _1h2o.fdt, _1h2o.fdx, _1h2o.fnm, _1h2o.nvd, _1h2o.nvm, _1h2o.si, _1h2o_LuceneVarGapDocFreqInterval_0.doc, _1h2o_LuceneVarGapDocFreqInterval_0.tib, _1h2o_LuceneVarGapDocFreqInterval_0.tiv, _1h2p.fdt, _1h2p.fdx, _1h2p.fnm, _1h2p.nvd, _1h2p.nvm, _1h2p.si, _1h2p_LuceneVarGapDocFreqInterval_0.doc, _1h2p_LuceneVarGapDocFreqInterval_0.tib, _1h2p_LuceneVarGapDocFreqInterval_0.tiv, _1h2q.fdt, _1h2q.fdx, _1h2q.fnm, _1h2q.nvd, _1h2q.nvm, _1h2q.si, _1h2q_LuceneVarGapDocFreqInterval_0.doc, _1h2q_LuceneVarGapDocFreqInterval_0.tib, _1h2q_LuceneVarGapDocFreqInterval_0.tiv, _1h2r.fdt, _1h2r.fdx, _1h2r.fnm, _1h2r.nvd, _1h2r.nvm, _1h2r.si, _1h2r_LuceneVarGapDocFreqInterval_0.doc, _1h2r_LuceneVarGapDocFreqInterval_0.tib, _1h2r_LuceneVarGapDocFreqInterval_0.tiv, _1h2s.fdt, _1h2s.fdx, _1h2s.fnm, _1h2s.nvd, _1h2s.nvm, _1h2s.si, _1h2s_LuceneVarGapDocFreqInterval_0.doc, _1h2s_LuceneVarGapDocFreqInterval_0.tib, _1h2s_LuceneVarGapDocFreqInterval_0.tiv, _1h2t.fdt, _1h2t.fdx, _1h2t.fnm, _1h2t.nvd, _1h2t.nvm, _1h2t.si, _1h2t_LuceneVarGapDocFreqInterval_0.doc, _1h2t_LuceneVarGapDocFreqInterval_0.tib, _1h2t_LuceneVarGapDocFreqInterval_0.tiv, _1h2u.fdt, _1h2u.fdx, _1h2u.fnm, _1h2u.nvd, _1h2u.nvm, _1h2u.si, _1h2u_LuceneVarGapDocFreqInterval_0.doc, _1h2u_LuceneVarGapDocFreqInterval_0.tib, _1h2u_LuceneVarGapDocFreqInterval_0.tiv, _1h2v.fdt, _1h2v.fdx, _1h2v.fnm, _1h2v.nvd, _1h2v.nvm, _1h2v.si, _1h2v_LuceneVarGapDocFreqInterval_0.doc, _1h2v_LuceneVarGapDocFreqInterval_0.tib, _1h2v_LuceneVarGapDocFreqInterval_0.tiv, _1h2w.fdt, _1h2w.fdx, _1h2w.fnm, _1h2w.nvd, _1h2w.nvm, _1h2w.si, _1h2w_LuceneVarGapDocFreqInterval_0.doc, _1h2w_LuceneVarGapDocFreqInterval_0.tib, _1h2w_LuceneVarGapDocFreqInterval_0.tiv, _1h2x.fdt, _1h2x.fdx, _1h2x.fnm, _1h2x.nvd, _1h2x.nvm, _1h2x.si, _1h2x_LuceneVarGapDocFreqInterval_0.doc, _1h2x_LuceneVarGapDocFreqInterval_0.tib, _1h2x_LuceneVarGapDocFreqInterval_0.tiv, _1h2y.fdt, _1h2y.fdx, _1h2y.fnm, _1h2y.nvd, _1h2y.nvm, _1h2y.si, _1h2y_LuceneVarGapDocFreqInterval_0.doc, _1h2y_LuceneVarGapDocFreqInterval_0.tib, _1h2y_LuceneVarGapDocFreqInterval_0.tiv, _1h2z.fdt, _1h2z.fdx, _1h2z.fnm, _1h2z.nvd, _1h2z.nvm, _1h2z.si, _1h2z_LuceneVarGapDocFreqInterval_0.doc, _1h2z_LuceneVarGapDocFreqInterval_0.tib, _1h2z_LuceneVarGapDocFreqInterval_0.tiv, _1h30.fdt, _1h30.fdx, _1h30.fnm, _1h30.nvd, _1h30.nvm, _1h30.si, _1h30_LuceneVarGapDocFreqInterval_0.doc, _1h30_LuceneVarGapDocFreqInterval_0.tib, _1h30_LuceneVarGapDocFreqInterval_0.tiv, _1h31.fdt, _1h31.fdx, _1h31.fnm, _1h31.nvd, _1h31.nvm, _1h31.si, _1h31_LuceneVarGapDocFreqInterval_0.doc, _1h31_LuceneVarGapDocFreqInterval_0.tib, _1h31_LuceneVarGapDocFreqInterval_0.tiv, _1h32.fdt, _1h32.fdx, _1h32.fnm, _1h32.nvd, _1h32.nvm, _1h32.si, _1h32_LuceneVarGapDocFreqInterval_0.doc, _1h32_LuceneVarGapDocFreqInterval_0.tib, _1h32_LuceneVarGapDocFreqInterval_0.tiv, _1h33.fdt, _1h33.fdx, _1h33.fnm, _1h33.nvd, _1h33.nvm, _1h33.si, _1h33_LuceneVarGapDocFreqInterval_0.doc, _1h33_LuceneVarGapDocFreqInterval_0.tib, _1h33_LuceneVarGapDocFreqInterval_0.tiv, _1h34.fdt, _1h34.fdx, _1h34.fnm, _1h34.nvd, _1h34.nvm, _1h34.si, _1h34_LuceneVarGapDocFreqInterval_0.doc, _1h34_LuceneVarGapDocFreqInterval_0.tib, _1h34_LuceneVarGapDocFreqInterval_0.tiv, _1h35.fdt, _1h35.fdx, _1h35.fnm, _1h35.nvd, _1h35.nvm, _1h35.si, _1h35_LuceneVarGapDocFreqInterval_0.doc, _1h35_LuceneVarGapDocFreqInterval_0.tib, _1h35_LuceneVarGapDocFreqInterval_0.tiv, _1h37.fdt, _1h37.fdx, _1h37.fnm, _1h37.nvd, _1h37.nvm, _1h37.si, _1h37_LuceneVarGapDocFreqInterval_0.doc, _1h37_LuceneVarGapDocFreqInterval_0.tib, _1h37_LuceneVarGapDocFreqInterval_0.tiv, _1h38.fdt, _1h38.fdx, _1h38.fnm, _1h38.nvd, _1h38.nvm, _1h38.si, _1h38_LuceneVarGapDocFreqInterval_0.doc, _1h38_LuceneVarGapDocFreqInterval_0.tib, _1h38_LuceneVarGapDocFreqInterval_0.tiv, _1h39.fdt, _1h39.fdx, _1h39.fnm, _1h39.nvd, _1h39.nvm, _1h39.si, _1h39_LuceneVarGapDocFreqInterval_0.doc, _1h39_LuceneVarGapDocFreqInterval_0.tib, _1h39_LuceneVarGapDocFreqInterval_0.tiv, _1h3a.fdt, _1h3a.fdx, _1h3a.fnm, _1h3a.nvd, _1h3a.nvm, _1h3a.si, _1h3a_LuceneVarGapDocFreqInterval_0.doc, _1h3a_LuceneVarGapDocFreqInterval_0.tib, _1h3a_LuceneVarGapDocFreqInterval_0.tiv, _1h3b.fdt, _1h3b.fdx, _1h3b.fnm, _1h3b.nvd, _1h3b.nvm, _1h3b.si, _1h3b_LuceneVarGapDocFreqInterval_0.doc, _1h3b_LuceneVarGapDocFreqInterval_0.tib, _1h3b_LuceneVarGapDocFreqInterval_0.tiv, _1h3c.fdt, _1h3c.fdx, _1h3c.fnm, _1h3c.nvd, _1h3c.nvm, _1h3c.si, _1h3c_LuceneVarGapDocFreqInterval_0.doc, _1h3c_LuceneVarGapDocFreqInterval_0.tib, _1h3c_LuceneVarGapDocFreqInterval_0.tiv, _1h3d.fdt, _1h3d.fdx, _1h3d.fnm, _1h3d.nvd, _1h3d.nvm, _1h3d.si, _1h3d_LuceneVarGapDocFreqInterval_0.doc, _1h3d_LuceneVarGapDocFreqInterval_0.tib, _1h3d_LuceneVarGapDocFreqInterval_0.tiv, _1h3e.fdt, _1h3e.fdx, _1h3e.fnm, _1h3e.nvd, _1h3e.nvm, _1h3e.si, _1h3e_LuceneVarGapDocFreqInterval_0.doc, _1h3e_LuceneVarGapDocFreqInterval_0.tib, _1h3e_LuceneVarGapDocFreqInterval_0.tiv, _1h3f.fdt, _1h3f.fdx, _1h3f.fnm, _1h3f.nvd, _1h3f.nvm, _1h3f.si, _1h3f_LuceneVarGapDocFreqInterval_0.doc, _1h3f_LuceneVarGapDocFreqInterval_0.tib, _1h3f_LuceneVarGapDocFreqInterval_0.tiv, _1h3g.fdt, _1h3g.fdx, _1h3g.fnm, _1h3g.nvd, _1h3g.nvm, _1h3g.si, _1h3g_LuceneVarGapDocFreqInterval_0.doc, _1h3g_LuceneVarGapDocFreqInterval_0.tib, _1h3g_LuceneVarGapDocFreqInterval_0.tiv, _1h3h.fdt, _1h3h.fdx, _1h3h.fnm, _1h3h.nvd, _1h3h.nvm, _1h3h.si, _1h3h_LuceneVarGapDocFreqInterval_0.doc, _1h3h_LuceneVarGapDocFreqInterval_0.tib, _1h3h_LuceneVarGapDocFreqInterval_0.tiv, _1h3i.fdt, _1h3i.fdx, _1h3i.fnm, _1h3i.nvd, _1h3i.nvm, _1h3i.si, _1h3i_LuceneVarGapDocFreqInterval_0.doc, _1h3i_LuceneVarGapDocFreqInterval_0.tib, _1h3i_LuceneVarGapDocFreqInterval_0.tiv, _1h3j.fdt, _1h3j.fdx, _1h3j.fnm, _1h3j.nvd, _1h3j.nvm, _1h3j.si, _1h3j_LuceneVarGapDocFreqInterval_0.doc, _1h3j_LuceneVarGapDocFreqInterval_0.tib, _1h3j_LuceneVarGapDocFreqInterval_0.tiv, _1h3k.fdt, _1h3k.fdx, _1h3k.fnm, _1h3k.nvd, _1h3k.nvm, _1h3k.si, _1h3k_LuceneVarGapDocFreqInterval_0.doc, _1h3k_LuceneVarGapDocFreqInterval_0.tib, _1h3k_LuceneVarGapDocFreqInterval_0.tiv, _su9.fdt, _su9.fdx, _su9.fnm, _su9.nvd, _su9.nvm, _su9.si, _su9_LuceneVarGapDocFreqInterval_0.doc, _su9_LuceneVarGapDocFreqInterval_0.tib, _su9_LuceneVarGapDocFreqInterval_0.tiv, segments_2]}]>
	at __randomizedtesting.SeedInfo.seed([6A03F0A0A3FD6B7D:4FD4EB90D3B5657E]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.failNotEquals(Assert.java:647)
	at org.junit.Assert.assertEquals(Assert.java:128)
	at org.junit.Assert.assertEquals(Assert.java:147)
	at org.apache.solr.handler.TestReplicationHandler.doTestReplicateAfterCoreReload(TestReplicationHandler.java:1137)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:10767/c8n_1x2_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:10767/c8n_1x2_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([6A03F0A0A3FD6B7D:E257CF7A0D010685]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:597)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:920)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:811)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:754)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:10767/c8n_1x2_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:569)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:211)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:207)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:594)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:461)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:47322/c8n_1x3_lf_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:47322/c8n_1x3_lf_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([6A03F0A0A3FD6B7D:E257CF7A0D010685]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:597)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:920)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:811)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:754)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:47322/c8n_1x3_lf_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:569)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:211)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:207)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:594)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:461)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:10849/c8n_1x3_commits_shard1_replica3

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:10849/c8n_1x3_commits_shard1_replica3
	at __randomizedtesting.SeedInfo.seed([6A03F0A0A3FD6B7D:E257CF7A0D010685]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:569)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:211)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:207)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:483)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:464)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:461)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33012/zz/ip/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33012/zz/ip/collection1
	at __randomizedtesting.SeedInfo.seed([6A03F0A0A3FD6B7D:E257CF7A0D010685]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:597)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:920)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:811)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:754)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:135)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:174)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:139)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:153)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:33012/zz/ip/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:569)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:211)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:207)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:594)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:461)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:41248/t_/w/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:41248/t_/w/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([6A03F0A0A3FD6B7D:E257CF7A0D010685]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:597)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:920)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:811)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:754)
	at org.apache.solr.client.solrj.SolrClient.request(SolrClient.java:1220)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carro

[...truncated too long message...]

tion1]  CLOSING SolrCore org.apache.solr.core.SolrCore@6321d18c
   [junit4]   2> 4395803 T2402 oasc.ZkController.unRegisterConfListener  a listener was removed because of core close
   [junit4]   2> 4395804 T2402 oasu.DirectUpdateHandler2.close closing DirectUpdateHandler2{commits=33,autocommits=0,soft autocommits=0,optimizes=0,rollbacks=0,expungeDeletes=0,docsPending=0,adds=0,deletesById=0,deletesByQuery=0,errors=0,cumulative_adds=144,cumulative_deletesById=70,cumulative_deletesByQuery=0,cumulative_errors=0,transaction_logs_total_size=8820,transaction_logs_total_number=1}
   [junit4]   2> 4395804 T2402 oasu.SolrCoreState.decrefSolrCoreState Closing SolrCoreState
   [junit4]   2> 4395805 T2402 oasu.DefaultSolrCoreState.closeIndexWriter SolrCoreState ref count has reached 0 - closing IndexWriter
   [junit4]   2> 4395805 T2402 oasu.DefaultSolrCoreState.closeIndexWriter closing IndexWriter with IndexWriterCloser
   [junit4]   2> 4395807 T2402 oasc.SolrCore.closeSearcher [collection1] Closing main searcher on request.
   [junit4]   2> 4395808 T2388 oascc.ZkStateReader$3.process WARN ZooKeeper watch triggered, but Solr cannot talk to ZK
   [junit4]   2> 4395811 T2402 oasc.CachingDirectoryFactory.close Closing StandardDirectoryFactory - 2 directories currently being tracked
   [junit4]   2> 4395812 T2402 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest 6A03F0A0A3FD6B7D-001/shard-5-001/cores/collection1/data [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest 6A03F0A0A3FD6B7D-001/shard-5-001/cores/collection1/data;done=false>>]
   [junit4]   2> 4395812 T2402 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest 6A03F0A0A3FD6B7D-001/shard-5-001/cores/collection1/data
   [junit4]   2> 4395812 T2402 oasc.CachingDirectoryFactory.closeCacheValue looking to close /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest 6A03F0A0A3FD6B7D-001/shard-5-001/cores/collection1/data/index.20150305212058328 [CachedDir<<refCount=0;path=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest 6A03F0A0A3FD6B7D-001/shard-5-001/cores/collection1/data/index.20150305212058328;done=false>>]
   [junit4]   2> 4395812 T2402 oasc.CachingDirectoryFactory.close Closing directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest 6A03F0A0A3FD6B7D-001/shard-5-001/cores/collection1/data/index.20150305212058328
   [junit4]   2> 4395814 T1892 oejsh.ContextHandler.doStop stopped o.e.j.s.ServletContextHandler{/,null}
   [junit4]   2> 4396018 T1892 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:58399 58399
   [junit4]   2> 4396136 T1893 oasc.ZkTestServer.send4LetterWord connecting to 127.0.0.1:58399 58399
   [junit4]   2> 4396138 T1893 oasc.ZkTestServer$ZKServerMain.runFromConfig WARN Watch limit violations: 
   [junit4]   2> 	Maximum concurrent create/delete watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		14	/solr/clusterstate.json
   [junit4]   2> 		14	/solr/aliases.json
   [junit4]   2> 		13	/solr/configs/conf1
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent data watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		5	/solr/collections/collection1/leader_elect/shard1/election/93426077316546569-core_node2-n_0000000001
   [junit4]   2> 		5	/solr/overseer_elect/election/93426077316546569-127.0.0.1:36163_-n_0000000002
   [junit4]   2> 		3	/solr/collections/collection1/leader_elect/shard1/election/93426077316546567-core_node1-n_0000000000
   [junit4]   2> 		3	/solr/overseer_elect/election/93426077316546567-127.0.0.1:27817_-n_0000000001
   [junit4]   2> 		2	/solr/collections/collection1/leader_elect/shard1/election/93426077316546573-core_node4-n_0000000003
   [junit4]   2> 		2	/solr/overseer_elect/election/93426077316546573-127.0.0.1:18482_-n_0000000004
   [junit4]   2> 	
   [junit4]   2> 	Maximum concurrent children watches above limit:
   [junit4]   2> 	
   [junit4]   2> 		7	/solr/live_nodes
   [junit4]   2> 		4	/solr/overseer/collection-queue-work
   [junit4]   2> 		3	/solr/overseer/queue
   [junit4]   2> 	
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ChaosMonkeySafeLeaderTest -Dtests.method=test -Dtests.seed=6A03F0A0A3FD6B7D -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=it -Dtests.timezone=Etc/GMT-6 -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
   [junit4] FAILURE 3963s J1 | ChaosMonkeySafeLeaderTest.test <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: shard1 is not consistent.  Got 305 from https://127.0.0.1:27817/collection1lastClient and got 283 from https://127.0.0.1:37517/collection1
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([6A03F0A0A3FD6B7D:E257CF7A0D010685]:0)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1286)
   [junit4]    > 	at org.apache.solr.cloud.AbstractFullDistribZkTestBase.checkShardConsistency(AbstractFullDistribZkTestBase.java:1265)
   [junit4]    > 	at org.apache.solr.cloud.ChaosMonkeySafeLeaderTest.test(ChaosMonkeySafeLeaderTest.java:161)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:958)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:933)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> 4396146 T1892 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J1/temp/solr.cloud.ChaosMonkeySafeLeaderTest 6A03F0A0A3FD6B7D-001
   [junit4]   2> 3962716 T1891 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=DummyCompressingStoredFields(storedFieldsFormat=CompressingStoredFieldsFormat(compressionMode=DUMMY, chunkSize=3, maxDocsPerChunk=345, blockSize=5), termVectorsFormat=CompressingTermVectorsFormat(compressionMode=DUMMY, chunkSize=3, blockSize=5)), sim=RandomSimilarityProvider(queryNorm=false,coord=yes): {}, locale=it, timezone=Etc/GMT-6
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=167603640,total=308281344
   [junit4]   2> NOTE: All tests run in this JVM: [FieldAnalysisRequestHandlerTest, FieldMutatingUpdateProcessorTest, SchemaVersionSpecificBehaviorTest, TestStressVersions, OverseerRolesTest, DeleteReplicaTest, ResourceLoaderTest, TestSweetSpotSimilarityFactory, TestZkChroot, AtomicUpdatesTest, TestStressRecovery, TestConfig, TestSearchPerf, HdfsDirectoryFactoryTest, TestFastOutputStream, TestRecovery, PluginInfoTest, RequestLoggingTest, TestSolrConfigHandlerCloud, WordBreakSolrSpellCheckerTest, TestCoreDiscovery, TestOmitPositions, TestRecoveryHdfs, TestStandardQParsers, TestStressLucene, TestQueryUtils, IndexSchemaTest, TestSolrQueryParserDefaultOperatorResource, DistributedFacetPivotLargeTest, PreAnalyzedFieldTest, TestMaxScoreQueryParser, DistributedFacetPivotSmallTest, HdfsLockFactoryTest, DateRangeFieldTest, ZkControllerTest, CircularListTest, TestBadConfig, SuggesterTSTTest, EchoParamsTest, BlockCacheTest, TestFuzzyAnalyzedSuggestions, TestCollectionAPI, TestSolr4Spatial, DistributedQueryComponentOptimizationTest, ChaosMonkeySafeLeaderTest]
   [junit4] Completed on J1 in 3963.50s, 1 test, 1 failure <<< FAILURES!

[...truncated 50 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 470 suites, 1873 tests, 5 errors, 3 failures, 58 ignored (9 assumptions)

Total time: 169 minutes 40 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 714253841 bytes
Compression is 0.0%
Took 2 min 11 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 776 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/776/

6 tests failed.
FAILED:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:38259/yp/ar/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:38259/yp/ar/collection1
	at __randomizedtesting.SeedInfo.seed([576AE74E94A4E5E0:DF3ED8943A588818]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:131)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:309)
	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:224)
	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:166)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:671)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test(FullSolrCloudDistribCmdsTest.java:151)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:950)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:925)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketException: Broken pipe
	at java.net.SocketOutputStream.socketWrite0(Native Method)
	at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:113)
	at java.net.SocketOutputStream.write(SocketOutputStream.java:159)
	at org.apache.http.impl.io.AbstractSessionOutputBuffer.flushBuffer(AbstractSessionOutputBuffer.java:159)
	at org.apache.http.impl.io.AbstractSessionOutputBuffer.writeLine(AbstractSessionOutputBuffer.java:258)
	at org.apache.http.impl.io.HttpRequestWriter.writeHeadLine(HttpRequestWriter.java:59)
	at org.apache.http.impl.io.HttpRequestWriter.writeHeadLine(HttpRequestWriter.java:46)
	at org.apache.http.impl.io.AbstractMessageWriter.write(AbstractMessageWriter.java:109)
	at org.apache.http.impl.AbstractHttpClientConnection.sendRequestHeader(AbstractHttpClientConnection.java:254)
	at org.apache.http.impl.conn.DefaultClientConnection.sendRequestHeader(DefaultClientConnection.java:270)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.sendRequestHeader(ManagedClientConnectionImpl.java:209)
	at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:203)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:121)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 49 more


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59038/c8n_1x2_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59038/c8n_1x2_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([576AE74E94A4E5E0:DF3ED8943A588818]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:597)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:918)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:809)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:752)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:950)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:925)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:59038/c8n_1x2_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:594)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 53 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:23556/mq/c8n_1x3_lf_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:23556/mq/c8n_1x3_lf_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([576AE74E94A4E5E0:DF3ED8943A588818]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:597)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:918)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:809)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:752)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:950)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:925)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:23556/mq/c8n_1x3_lf_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:594)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 53 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:36862/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:36862/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([576AE74E94A4E5E0:DF3ED8943A588818]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:131)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:176)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:154)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:950)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:925)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 48 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:47602/ol_vit/cj/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:47602/ol_vit/cj/collection1
	at __randomizedtesting.SeedInfo.seed([576AE74E94A4E5E0:DF3ED8943A588818]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:597)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:918)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:809)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:752)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:131)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:110)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:950)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:925)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:47602/ol_vit/cj/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:594)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 53 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:47629/s/hq/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:47629/s/hq/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([576AE74E94A4E5E0:DF3ED8943A588818]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:597)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:918)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:809)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:752)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:950)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:925)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:47629/s/hq/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:594)
	... 47 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 52 more




Build Log:
[...truncated 9840 lines...]
   [junit4] Suite: org.apache.solr.cloud.HttpPartitionTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/init-core-data-001
   [junit4]   2> 129374 T456 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 129382 T456 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 129383 T457 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 129483 T456 oasc.ZkTestServer.run start zk server on port:59015
   [junit4]   2> 129484 T456 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 129485 T456 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 129490 T464 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@27adb06f name:ZooKeeperConnection Watcher:127.0.0.1:59015 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 129490 T456 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 129491 T456 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 129491 T456 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 129495 T456 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 129496 T456 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 129498 T467 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1634b1c8 name:ZooKeeperConnection Watcher:127.0.0.1:59015/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 129498 T456 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 129498 T456 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 129499 T456 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 129501 T456 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 129502 T456 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 129503 T456 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 129505 T456 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 129507 T456 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 129510 T456 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 129510 T456 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 129512 T456 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 129513 T456 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 129514 T456 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 129515 T456 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 129517 T456 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 129517 T456 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 129519 T456 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 129520 T456 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 129521 T456 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 129522 T456 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 129523 T456 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 129524 T456 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 129525 T456 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 129526 T456 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 129528 T456 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 129528 T456 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 129530 T456 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 129531 T456 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 130116 T3 oasu.ConcurrentLRUCache.finalize ERROR ConcurrentLRUCache was not destroyed prior to finalize(), indicates a bug -- POSSIBLE RESOURCE LEAK!!!
   [junit4]   2> 131108 T456 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/control-001/cores/collection1
   [junit4]   2> 131114 T456 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 131120 T456 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:59019
   [junit4]   2> 131120 T456 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3972aa3f
   [junit4]   2> 131122 T456 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 131122 T456 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/control-001
   [junit4]   2> 131123 T456 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/control-001/'
   [junit4]   2> 131180 T456 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/control-001/solr.xml
   [junit4]   2> 131256 T456 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/control-001/cores
   [junit4]   2> 131257 T456 oasc.CoreContainer.<init> New CoreContainer 831599066
   [junit4]   2> 131257 T456 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/control-001/]
   [junit4]   2> 131258 T456 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/control-001/lib
   [junit4]   2> 131259 T456 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/control-001/lib).
   [junit4]   2> 131275 T456 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 131276 T456 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 131276 T456 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 131277 T456 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 131277 T456 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 131277 T456 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 131278 T456 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 131278 T456 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 131279 T456 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 131279 T456 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 131280 T456 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 131280 T456 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 131281 T456 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 131281 T456 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 131282 T456 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 131282 T456 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:59015/solr
   [junit4]   2> 131283 T456 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 131283 T456 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 131284 T456 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 131286 T479 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@1f9407d0 name:ZooKeeperConnection Watcher:127.0.0.1:59015 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 131287 T456 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 131288 T456 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 131291 T456 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 131293 T482 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@72639eb7 name:ZooKeeperConnection Watcher:127.0.0.1:59015/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 131293 T456 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 131295 T456 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 131297 T456 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 131299 T456 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 131302 T456 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 131304 T456 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 131307 T456 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 131308 T456 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59018_
   [junit4]   2> 131309 T456 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:59018_
   [junit4]   2> 131311 T456 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 131312 T456 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 131314 T456 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 131316 T456 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:59018_
   [junit4]   2> 131316 T456 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 131318 T456 oasc.Overseer.start Overseer (id=93420428468420611-127.0.0.1:59018_-n_0000000000) starting
   [junit4]   2> 131320 T456 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 131326 T456 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 131327 T484 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 131328 T483 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 131328 T456 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 131331 T456 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 131332 T456 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 131336 T456 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/control-001/cores
   [junit4]   2> 131339 T456 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/control-001/cores/collection1/
   [junit4]   2> 131340 T456 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 131342 T486 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 131342 T486 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 131343 T486 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 131343 T482 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 131345 T483 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:59018_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59018",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null} current state version: 0
   [junit4]   2> 131345 T483 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:59018_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59018",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "shard":null}
   [junit4]   2> 131346 T483 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 131346 T483 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 131348 T482 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 132344 T486 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 132345 T486 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 132346 T486 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 132346 T486 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 132347 T486 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 132347 T486 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/control-001/cores/collection1/'
   [junit4]   2> 132365 T486 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 132368 T486 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 132380 T486 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 132403 T486 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 132452 T486 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 132453 T486 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 132465 T486 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 132892 T486 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 132905 T486 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 132908 T486 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 132924 T486 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 132929 T486 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 132934 T486 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 132935 T486 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 132935 T486 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 132936 T486 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 132937 T486 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 132937 T486 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 132937 T486 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 132938 T486 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 132938 T486 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/tempDir-001/control/data/
   [junit4]   2> 132939 T486 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@64115bbc
   [junit4]   2> 132940 T486 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/tempDir-001/control/data
   [junit4]   2> 132940 T486 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/tempDir-001/control/data/index/
   [junit4]   2> 132941 T486 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 132941 T486 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/tempDir-001/control/data/index
   [junit4]   2> 132942 T486 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.LogByteSizeMergePolicy: [LogByteSizeMergePolicy: minMergeSize=1677721, mergeFactor=6, maxMergeSize=2147483648, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7481991336623879]
   [junit4]   2> 132943 T486 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1796018a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@793f171e),segFN=segments_1,generation=1}
   [junit4]   2> 132943 T486 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 132948 T486 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 132949 T486 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 132949 T486 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 132950 T486 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 132950 T486 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 132950 T486 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 132951 T486 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 132951 T486 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 132952 T486 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 132953 T486 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 132953 T486 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 132954 T486 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 132955 T486 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 132955 T486 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 132956 T486 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 132956 T486 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 132957 T486 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 132958 T486 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 132959 T486 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 132959 T486 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 132960 T486 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 132961 T486 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 132961 T486 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 132962 T486 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 132963 T486 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 132963 T486 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 132964 T486 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 132987 T486 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 132989 T486 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 132991 T486 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 132994 T486 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 132996 T486 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 132998 T486 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 133000 T486 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 133001 T486 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 133001 T486 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@edd5a57
   [junit4]   2> 133002 T486 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@1796018a lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@793f171e),segFN=segments_1,generation=1}
   [junit4]   2> 133003 T486 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 133003 T486 oass.SolrIndexSearcher.<init> Opening Searcher@6ce139a0[collection1] main
   [junit4]   2> 133004 T486 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 133005 T486 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 133005 T486 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 133005 T486 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 133006 T486 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 133006 T486 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 133007 T486 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 133007 T486 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 133008 T486 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 133008 T486 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 133009 T487 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6ce139a0[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 133009 T486 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 133010 T490 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:59018 collection:control_collection shard:shard1
   [junit4]   2> 133011 T456 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0
   [junit4]   2> 133012 T456 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 133012 T490 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 133015 T456 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 133015 T456 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 133017 T490 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 133018 T494 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@34fc91f name:ZooKeeperConnection Watcher:127.0.0.1:59015/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 133018 T456 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 133018 T482 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 133019 T456 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 133018 T490 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 133021 T490 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> 133021 T483 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 133021 T456 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> ASYNC  NEW_CORE C181 name=collection1 org.apache.solr.core.SolrCore@78967034 url=http://127.0.0.1:59018/collection1 node=127.0.0.1:59018_ C181_STATE=coll:control_collection core:collection1 props:{state=down, node_name=127.0.0.1:59018_, core=collection1, base_url=http://127.0.0.1:59018}
   [junit4]   2> 133021 T490 C181 P59018 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:59018/collection1/
   [junit4]   2> 133022 T490 C181 P59018 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 133022 T490 C181 P59018 oasc.SyncStrategy.syncToMe http://127.0.0.1:59018/collection1/ has no replicas
   [junit4]   2> 133022 T482 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 133022 T494 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 0)
   [junit4]   2> 133022 T490 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:59018/collection1/ shard1
   [junit4]   2> 133024 T490 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 133024 T456 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 133025 T456 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 133025 T456 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 133026 T456 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 133027 T497 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@efd1d8c name:ZooKeeperConnection Watcher:127.0.0.1:59015/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 133027 T456 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 133027 T482 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 133028 T456 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 133029 T483 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59018",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 133032 T483 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 133032 T483 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 133033 T482 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 133033 T494 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 133035 T482 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 133036 T483 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 133037 T482 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 133037 T494 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 133078 T490 oasc.ZkController.register We are http://127.0.0.1:59018/collection1/ and leader is http://127.0.0.1:59018/collection1/
   [junit4]   2> 133079 T490 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:59018
   [junit4]   2> 133079 T490 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 133080 T490 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 133080 T490 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 133081 T482 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 133082 T483 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:59018_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59018",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":"shard1"} current state version: 4
   [junit4]   2> 133083 T483 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "node_name":"127.0.0.1:59018_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:59018",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "shard":"shard1"}
   [junit4]   2> 133186 T482 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 133186 T494 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 134178 T456 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/shard-1-001/cores/collection1
   [junit4]   2> 134181 T456 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 134182 T456 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 134184 T456 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:59033
   [junit4]   2> 134185 T456 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3972aa3f
   [junit4]   2> 134186 T456 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 134186 T456 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/shard-1-001
   [junit4]   2> 134186 T456 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/shard-1-001/'
   [junit4]   2> 134222 T456 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/shard-1-001/solr.xml
   [junit4]   2> 134266 T456 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/shard-1-001/cores
   [junit4]   2> 134266 T456 oasc.CoreContainer.<init> New CoreContainer 175186298
   [junit4]   2> 134266 T456 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/shard-1-001/]
   [junit4]   2> 134267 T456 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/shard-1-001/lib
   [junit4]   2> 134267 T456 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/shard-1-001/lib).
   [junit4]   2> 134278 T456 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 134278 T456 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 134278 T456 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 134279 T456 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 134279 T456 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 134279 T456 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 134280 T456 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 134280 T456 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 134280 T456 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 134280 T456 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 134281 T456 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 134281 T456 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 134282 T456 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 134282 T456 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 134282 T456 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 134283 T456 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:59015/solr
   [junit4]   2> 134283 T456 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 134283 T456 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 134284 T456 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 134286 T509 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6014ddbc name:ZooKeeperConnection Watcher:127.0.0.1:59015 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 134287 T456 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 134287 T456 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 134290 T456 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 134292 T512 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7a095640 name:ZooKeeperConnection Watcher:127.0.0.1:59015/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 134292 T456 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 134298 T456 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 135303 T456 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:59032_
   [junit4]   2> 135304 T456 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:59032_
   [junit4]   2> 135307 T456 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 135309 T456 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/shard-1-001/cores
   [junit4]   2> 135310 T456 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.HttpPartitionTest 576AE74E94A4E5E0-001/shard-1-001/cores/collection1/
   [junit4]   2> 135311 T456 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 135313 T513 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 135314 T513 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 135315 T482 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 135315 T513 oasc.ZkController.preRegister Registering watch for external collection collection1
   [junit4]   2> 135316 T513 oascc.ZkStateReader.addZkWatch addZkWatch collection1
   [junit4]   2> 135316 T483 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "node_name":"127.0.0.1:59032_",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "base_url

[...truncated too long message...]

king to server at: http://127.0.0.1:47629/s/hq/repfacttest_c8n_1x3_shard1_replica2
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([576AE74E94A4E5E0:DF3ED8943A588818]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:597)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:918)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:809)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:752)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:950)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:925)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:47629/s/hq/repfacttest_c8n_1x3_shard1_replica2
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:594)
   [junit4]    > 	... 47 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	... 52 more
   [junit4]   2> 2019003 T11425 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J0/temp/solr.cloud.ReplicationFactorTest 576AE74E94A4E5E0-001
   [junit4]   2> 71272 T11424 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {multiDefault=FSTOrd50, range_facet_l=PostingsFormat(name=LuceneVarGapFixedInterval), intDefault=PostingsFormat(name=LuceneVarGapFixedInterval), text=Lucene50(blocksize=128), id=PostingsFormat(name=Memory doPackFST= true), a_t=PostingsFormat(name=LuceneVarGapFixedInterval), _version_=PostingsFormat(name=LuceneVarGapFixedInterval), timestamp=PostingsFormat(name=LuceneVarGapFixedInterval)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=DefaultSimilarity, locale=et, timezone=America/Argentina/Buenos_Aires
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=194786384,total=501743616
   [junit4]   2> NOTE: All tests run in this JVM: [MultiThreadedOCPTest, TestFastLRUCache, HttpPartitionTest, TestSolrIndexConfig, TestSolrDeletionPolicy2, DocExpirationUpdateProcessorFactoryTest, HdfsDirectoryTest, TimeZoneUtilsTest, FastVectorHighlighterTest, BasicFunctionalityTest, TestManagedResource, DistributedQueryElevationComponentTest, PrimUtilsTest, TestOverriddenPrefixQueryForCustomFieldType, TestObjectReleaseTracker, TestTrie, TestUniqueKeyFieldResource, TestExactSharedStatsCache, TestAtomicUpdateErrorCases, TestReplicaProperties, TestIBSimilarityFactory, TestConfigSets, HdfsWriteToMultipleCollectionsTest, TestImplicitCoreProperties, ExternalFileFieldSortTest, TestComplexPhraseQParserPlugin, TestOrdValues, TestNonDefinedSimilarityFactory, BasicDistributedZkTest, ZkSolrClientTest, TestCollationFieldDocValues, SimpleFacetsTest, OverseerTest, RequestHandlersTest, DistanceFunctionTest, TestFileDictionaryLookup, TestFieldTypeResource, TestStressReorder, SearchHandlerTest, TestBlendedInfixSuggestions, ZkNodePropsTest, TestManagedResourceStorage, TestFaceting, InfoHandlerTest, TestCopyFieldCollectionResource, ZkStateWriterTest, TestDocSet, RegexBoostProcessorTest, DirectSolrSpellCheckerTest, HighlighterConfigTest, TestNRTOpen, MBeansHandlerTest, ShowFileRequestHandlerTest, SpatialHeatmapFacetsTest, StatelessScriptUpdateProcessorFactoryTest, TestXIncludeConfig, TestSolrJ, TestCharFilters, CollectionsAPIDistributedZkTest, ShardRoutingTest, RecoveryZkTest, TestRandomFaceting, LeaderElectionTest, TestRandomDVFaceting, ShardRoutingCustomTest, AutoCommitTest, TestDistributedGrouping, TestHashPartitioner, DistributedSpellCheckComponentTest, TestCoreContainer, TestSolr4Spatial, SpellCheckComponentTest, SolrCmdDistributorTest, BadIndexSchemaTest, TestSort, TestIndexSearcher, CurrencyFieldOpenExchangeTest, CurrencyFieldXmlFileTest, SolrIndexSplitterTest, SimplePostToolTest, SignatureUpdateProcessorFactoryTest, CoreAdminHandlerTest, SuggesterTest, PolyFieldTest, WordBreakSolrSpellCheckerTest, TestWordDelimiterFilterFactory, CacheHeaderTest, DisMaxRequestHandlerTest, TestWriterPerf, DocumentAnalysisRequestHandlerTest, XmlUpdateRequestHandlerTest, PathHierarchyTokenizerFactoryTest, TestSolrQueryParser, IndexSchemaRuntimeFieldTest, TestCollationField, JSONWriterTest, TestAnalyzedSuggestions, JsonLoaderTest, TestBinaryResponseWriter, AlternateDirectoryTest, BadComponentTest, SampleTest, TestBinaryField, MinimalSchemaTest, OutputWriterTest, TestFuzzyAnalyzedSuggestions, TestPostingsSolrHighlighter, TestSweetSpotSimilarityFactory, TestPerFieldSimilarity, TestBM25SimilarityFactory, ResourceLoaderTest, URLClassifyProcessorTest, DOMUtilTest, ClusterStateTest, TestLRUCache, SliceStateTest, UUIDFieldTest, TestRTGBase, CursorPagingTest, SolrTestCaseJ4Test, TestCursorMarkWithoutUniqueKey, TestHighlightDedupGrouping, TestTolerantSearch, ActionThrottleTest, AliasIntegrationTest, CloudExitableDirectoryReaderTest, ConnectionManagerTest, CustomCollectionTest, DeleteReplicaTest, DistribDocExpirationUpdateProcessorTest, LeaderInitiatedRecoveryOnCommitTest, ReplicationFactorTest]
   [junit4] Completed on J0 in 71.57s, 1 test, 1 error <<< FAILURES!

[...truncated 345 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 469 suites, 1872 tests, 6 errors, 58 ignored (9 assumptions)

Total time: 155 minutes 35 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 673815956 bytes
Compression is 0.0%
Took 2 min 6 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 775 - Still Failing

Posted by Adrien Grand <jp...@gmail.com>.
I tried to explain the problem and proposed a small patch on
https://issues.apache.org/jira/browse/LUCENE-6329

On Tue, Mar 3, 2015 at 6:05 PM, Robert Muir <rc...@gmail.com> wrote:
> Well, we should think about the api and not rush it.
>
> i was just thinking that right now, this problem is really a
> disjunctions one, in that it does something wrong if you call score(),
> even though you said you didnt need them. but in this case, if it
> wrapped its subs with constantscorer, it would then be "well-behaved"
> ?
>
> On Tue, Mar 3, 2015 at 12:02 PM, Adrien Grand <jp...@gmail.com> wrote:
>> OK, it works for me too. Then I'll also add assertions that you do not
>> call score() if you passed needsScores=false.
>>
>> On Tue, Mar 3, 2015 at 5:38 PM, Robert Muir <rc...@gmail.com> wrote:
>>> one option is to wrap with constantscorer somewhere and still keep the
>>> optimization?
>>>
>>> On Tue, Mar 3, 2015 at 11:32 AM, Adrien Grand <jp...@gmail.com> wrote:
>>>> OK, I found the reason. We have an optimization in disjunctions to
>>>> stop calling matches() after a single matching clause has been found
>>>> if scores are not needed, but it does not play well with the fact that
>>>> some scorers always call score() on the underlying clauses even if
>>>> scores are not needed (BooleanScorer in this case, maybe there are
>>>> other ones). So I'll disable the optimization. We'll need to better
>>>> document when it is legal to call score() if we want to bring it back.
>>>>
>>>> On Tue, Mar 3, 2015 at 3:57 PM, Adrien Grand <jp...@gmail.com> wrote:
>>>>> I'm looking into it.
>>>>>
>>>>> On Tue, Mar 3, 2015 at 3:52 PM, Apache Jenkins Server
>>>>> <je...@builds.apache.org> wrote:
>>>>>> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/775/
>>>>>>
>>>>>> 1 tests failed.
>>>>>> FAILED:  org.apache.lucene.search.TestLRUQueryCache.testRandom
>>>>>>
>>>>>> Error Message:
>>>>>>
>>>>>>
>>>>>> Stack Trace:
>>>>>> java.lang.AssertionError
>>>>>>         at __randomizedtesting.SeedInfo.seed([8AFB9C1E9D5CF8AF:F8B7B9112C3C4EDC]:0)
>>>>>>         at org.apache.lucene.search.TermScorer.score(TermScorer.java:70)
>>>>>>         at org.apache.lucene.search.DisjunctionSumScorer.score(DisjunctionSumScorer.java:46)
>>>>>>         at org.apache.lucene.search.DisjunctionScorer.score(DisjunctionScorer.java:219)
>>>>>>         at org.apache.lucene.search.ReqExclScorer.score(ReqExclScorer.java:121)
>>>>>>         at org.apache.lucene.search.BooleanScorer$OrCollector.collect(BooleanScorer.java:142)
>>>>>>         at org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:181)
>>>>>>         at org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:171)
>>>>>>         at org.apache.lucene.search.BooleanScorer$BulkScorerAndDoc.score(BooleanScorer.java:60)
>>>>>>         at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:210)
>>>>>>         at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:264)
>>>>>>         at org.apache.lucene.search.BooleanScorer.score(BooleanScorer.java:289)
>>>>>>         at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:79)
>>>>>>         at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:63)
>>>>>>         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:694)
>>>>>>         at org.apache.lucene.search.AssertingIndexSearcher.search(AssertingIndexSearcher.java:102)
>>>>>>         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:408)
>>>>>>         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:617)
>>>>>>         at org.apache.lucene.search.IndexSearcher.count(IndexSearcher.java:290)
>>>>>>         at org.apache.lucene.search.TestLRUQueryCache.testRandom(TestLRUQueryCache.java:923)
>>>>>>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>>>>>>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>>>>>>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>>>>>         at java.lang.reflect.Method.invoke(Method.java:606)
>>>>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
>>>>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
>>>>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
>>>>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
>>>>>>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>>>>>>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>>>>>>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>>>>>>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>>>>>>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>>>>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>>>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>>>>>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>>>>>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>>>>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
>>>>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
>>>>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
>>>>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
>>>>>>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>>>>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>>>>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>>>>>>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>>>>>>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>>>>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>>>>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>>>>>>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>>>>>>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>>>>>>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>>>>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>>>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>>>>>>         at java.lang.Thread.run(Thread.java:745)
>>>>>>
>>>>>>
>>>>>>
>>>>>>
>>>>>> Build Log:
>>>>>> [...truncated 839 lines...]
>>>>>>    [junit4] Suite: org.apache.lucene.search.TestLRUQueryCache
>>>>>>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>>>>>>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLRUQueryCache -Dtests.method=testRandom -Dtests.seed=8AFB9C1E9D5CF8AF -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=en_ZA -Dtests.timezone=Africa/Porto-Novo -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
>>>>>>    [junit4] FAILURE 19.5s J3 | TestLRUQueryCache.testRandom <<<
>>>>>>    [junit4]    > Throwable #1: java.lang.AssertionError
>>>>>>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([8AFB9C1E9D5CF8AF:F8B7B9112C3C4EDC]:0)
>>>>>>    [junit4]    >        at org.apache.lucene.search.TermScorer.score(TermScorer.java:70)
>>>>>>    [junit4]    >        at org.apache.lucene.search.DisjunctionSumScorer.score(DisjunctionSumScorer.java:46)
>>>>>>    [junit4]    >        at org.apache.lucene.search.DisjunctionScorer.score(DisjunctionScorer.java:219)
>>>>>>    [junit4]    >        at org.apache.lucene.search.ReqExclScorer.score(ReqExclScorer.java:121)
>>>>>>    [junit4]    >        at org.apache.lucene.search.BooleanScorer$OrCollector.collect(BooleanScorer.java:142)
>>>>>>    [junit4]    >        at org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:181)
>>>>>>    [junit4]    >        at org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:171)
>>>>>>    [junit4]    >        at org.apache.lucene.search.BooleanScorer$BulkScorerAndDoc.score(BooleanScorer.java:60)
>>>>>>    [junit4]    >        at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:210)
>>>>>>    [junit4]    >        at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:264)
>>>>>>    [junit4]    >        at org.apache.lucene.search.BooleanScorer.score(BooleanScorer.java:289)
>>>>>>    [junit4]    >        at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:79)
>>>>>>    [junit4]    >        at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:63)
>>>>>>    [junit4]    >        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:694)
>>>>>>    [junit4]    >        at org.apache.lucene.search.AssertingIndexSearcher.search(AssertingIndexSearcher.java:102)
>>>>>>    [junit4]    >        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:408)
>>>>>>    [junit4]    >        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:617)
>>>>>>    [junit4]    >        at org.apache.lucene.search.IndexSearcher.count(IndexSearcher.java:290)
>>>>>>    [junit4]    >        at org.apache.lucene.search.TestLRUQueryCache.testRandom(TestLRUQueryCache.java:923)
>>>>>>    [junit4]    >        at java.lang.Thread.run(Thread.java:745)
>>>>>>    [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestLRUQueryCache 8AFB9C1E9D5CF8AF-001
>>>>>>    [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50), sim=DefaultSimilarity, locale=en_ZA, timezone=Africa/Porto-Novo
>>>>>>    [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=148303864,total=313524224
>>>>>>    [junit4]   2> NOTE: All tests run in this JVM: [FuzzyTermOnShortTermsTest, TestByteSlices, TestEarlyTermination, TestRollback, TestWildcard, TestDocCount, TestMmapDirectory, TestSpanExplanationsOfNonMatches, TestBagOfPositions, TestInPlaceMergeSorter, TestSparseFixedBitSet, TestAutomaton, Nested1, TestFilteredQuery, TestLucene50TermVectorsFormat, TestDocValuesTermsQuery, TestDocBoost, TestPhraseQuery, TestWindowsMMap, TestFastDecompressionMode, TestMultiLevelSkipList, TestCachingWrapperQuery, TestMatchAllDocsQuery, TestNamedSPILoader, TestBytesRefHash, TestPerFieldPostingsFormat2, TestForUtil, TestIndexWriterForceMerge, TestIndexableField, TestLock, TestMaxTermFrequency, TestIndexWriterOnJRECrash, TestLRUQueryCache]
>>>>>>    [junit4] Completed on J3 in 25.65s, 12 tests, 1 failure <<< FAILURES!
>>>>>>
>>>>>> [...truncated 995 lines...]
>>>>>> BUILD FAILED
>>>>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
>>>>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
>>>>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
>>>>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
>>>>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:49: The following error occurred while executing this line:
>>>>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
>>>>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 419 suites, 3370 tests, 1 failure, 39 ignored (29 assumptions)
>>>>>>
>>>>>> Total time: 48 minutes 1 second
>>>>>> Build step 'Invoke Ant' marked build as failure
>>>>>> Archiving artifacts
>>>>>> Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
>>>>>> Archived 5 artifacts
>>>>>> Archive block size is 32768
>>>>>> Received 0 blocks and 3613290 bytes
>>>>>> Compression is 0.0%
>>>>>> Took 1 sec
>>>>>> Recording test results
>>>>>> Email was triggered for: Failure
>>>>>> Sending email for trigger: Failure
>>>>>>
>>>>>>
>>>>>>
>>>>>>
>>>>>> ---------------------------------------------------------------------
>>>>>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>>>>>> For additional commands, e-mail: dev-help@lucene.apache.org
>>>>>
>>>>>
>>>>>
>>>>> --
>>>>> Adrien
>>>>
>>>>
>>>>
>>>> --
>>>> Adrien
>>>>
>>>> ---------------------------------------------------------------------
>>>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>>>> For additional commands, e-mail: dev-help@lucene.apache.org
>>>>
>>>
>>> ---------------------------------------------------------------------
>>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>>> For additional commands, e-mail: dev-help@lucene.apache.org
>>>
>>
>>
>>
>> --
>> Adrien
>>
>> ---------------------------------------------------------------------
>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>> For additional commands, e-mail: dev-help@lucene.apache.org
>>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>



-- 
Adrien

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


Re: [JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 775 - Still Failing

Posted by Robert Muir <rc...@gmail.com>.
Well, we should think about the api and not rush it.

i was just thinking that right now, this problem is really a
disjunctions one, in that it does something wrong if you call score(),
even though you said you didnt need them. but in this case, if it
wrapped its subs with constantscorer, it would then be "well-behaved"
?

On Tue, Mar 3, 2015 at 12:02 PM, Adrien Grand <jp...@gmail.com> wrote:
> OK, it works for me too. Then I'll also add assertions that you do not
> call score() if you passed needsScores=false.
>
> On Tue, Mar 3, 2015 at 5:38 PM, Robert Muir <rc...@gmail.com> wrote:
>> one option is to wrap with constantscorer somewhere and still keep the
>> optimization?
>>
>> On Tue, Mar 3, 2015 at 11:32 AM, Adrien Grand <jp...@gmail.com> wrote:
>>> OK, I found the reason. We have an optimization in disjunctions to
>>> stop calling matches() after a single matching clause has been found
>>> if scores are not needed, but it does not play well with the fact that
>>> some scorers always call score() on the underlying clauses even if
>>> scores are not needed (BooleanScorer in this case, maybe there are
>>> other ones). So I'll disable the optimization. We'll need to better
>>> document when it is legal to call score() if we want to bring it back.
>>>
>>> On Tue, Mar 3, 2015 at 3:57 PM, Adrien Grand <jp...@gmail.com> wrote:
>>>> I'm looking into it.
>>>>
>>>> On Tue, Mar 3, 2015 at 3:52 PM, Apache Jenkins Server
>>>> <je...@builds.apache.org> wrote:
>>>>> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/775/
>>>>>
>>>>> 1 tests failed.
>>>>> FAILED:  org.apache.lucene.search.TestLRUQueryCache.testRandom
>>>>>
>>>>> Error Message:
>>>>>
>>>>>
>>>>> Stack Trace:
>>>>> java.lang.AssertionError
>>>>>         at __randomizedtesting.SeedInfo.seed([8AFB9C1E9D5CF8AF:F8B7B9112C3C4EDC]:0)
>>>>>         at org.apache.lucene.search.TermScorer.score(TermScorer.java:70)
>>>>>         at org.apache.lucene.search.DisjunctionSumScorer.score(DisjunctionSumScorer.java:46)
>>>>>         at org.apache.lucene.search.DisjunctionScorer.score(DisjunctionScorer.java:219)
>>>>>         at org.apache.lucene.search.ReqExclScorer.score(ReqExclScorer.java:121)
>>>>>         at org.apache.lucene.search.BooleanScorer$OrCollector.collect(BooleanScorer.java:142)
>>>>>         at org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:181)
>>>>>         at org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:171)
>>>>>         at org.apache.lucene.search.BooleanScorer$BulkScorerAndDoc.score(BooleanScorer.java:60)
>>>>>         at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:210)
>>>>>         at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:264)
>>>>>         at org.apache.lucene.search.BooleanScorer.score(BooleanScorer.java:289)
>>>>>         at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:79)
>>>>>         at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:63)
>>>>>         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:694)
>>>>>         at org.apache.lucene.search.AssertingIndexSearcher.search(AssertingIndexSearcher.java:102)
>>>>>         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:408)
>>>>>         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:617)
>>>>>         at org.apache.lucene.search.IndexSearcher.count(IndexSearcher.java:290)
>>>>>         at org.apache.lucene.search.TestLRUQueryCache.testRandom(TestLRUQueryCache.java:923)
>>>>>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>>>>>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>>>>>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>>>>         at java.lang.reflect.Method.invoke(Method.java:606)
>>>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
>>>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
>>>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
>>>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
>>>>>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>>>>>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>>>>>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>>>>>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>>>>>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>>>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>>>>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>>>>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>>>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
>>>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
>>>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
>>>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
>>>>>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>>>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>>>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>>>>>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>>>>>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>>>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>>>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>>>>>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>>>>>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>>>>>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>>>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>>>>>         at java.lang.Thread.run(Thread.java:745)
>>>>>
>>>>>
>>>>>
>>>>>
>>>>> Build Log:
>>>>> [...truncated 839 lines...]
>>>>>    [junit4] Suite: org.apache.lucene.search.TestLRUQueryCache
>>>>>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>>>>>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLRUQueryCache -Dtests.method=testRandom -Dtests.seed=8AFB9C1E9D5CF8AF -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=en_ZA -Dtests.timezone=Africa/Porto-Novo -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
>>>>>    [junit4] FAILURE 19.5s J3 | TestLRUQueryCache.testRandom <<<
>>>>>    [junit4]    > Throwable #1: java.lang.AssertionError
>>>>>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([8AFB9C1E9D5CF8AF:F8B7B9112C3C4EDC]:0)
>>>>>    [junit4]    >        at org.apache.lucene.search.TermScorer.score(TermScorer.java:70)
>>>>>    [junit4]    >        at org.apache.lucene.search.DisjunctionSumScorer.score(DisjunctionSumScorer.java:46)
>>>>>    [junit4]    >        at org.apache.lucene.search.DisjunctionScorer.score(DisjunctionScorer.java:219)
>>>>>    [junit4]    >        at org.apache.lucene.search.ReqExclScorer.score(ReqExclScorer.java:121)
>>>>>    [junit4]    >        at org.apache.lucene.search.BooleanScorer$OrCollector.collect(BooleanScorer.java:142)
>>>>>    [junit4]    >        at org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:181)
>>>>>    [junit4]    >        at org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:171)
>>>>>    [junit4]    >        at org.apache.lucene.search.BooleanScorer$BulkScorerAndDoc.score(BooleanScorer.java:60)
>>>>>    [junit4]    >        at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:210)
>>>>>    [junit4]    >        at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:264)
>>>>>    [junit4]    >        at org.apache.lucene.search.BooleanScorer.score(BooleanScorer.java:289)
>>>>>    [junit4]    >        at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:79)
>>>>>    [junit4]    >        at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:63)
>>>>>    [junit4]    >        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:694)
>>>>>    [junit4]    >        at org.apache.lucene.search.AssertingIndexSearcher.search(AssertingIndexSearcher.java:102)
>>>>>    [junit4]    >        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:408)
>>>>>    [junit4]    >        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:617)
>>>>>    [junit4]    >        at org.apache.lucene.search.IndexSearcher.count(IndexSearcher.java:290)
>>>>>    [junit4]    >        at org.apache.lucene.search.TestLRUQueryCache.testRandom(TestLRUQueryCache.java:923)
>>>>>    [junit4]    >        at java.lang.Thread.run(Thread.java:745)
>>>>>    [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestLRUQueryCache 8AFB9C1E9D5CF8AF-001
>>>>>    [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50), sim=DefaultSimilarity, locale=en_ZA, timezone=Africa/Porto-Novo
>>>>>    [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=148303864,total=313524224
>>>>>    [junit4]   2> NOTE: All tests run in this JVM: [FuzzyTermOnShortTermsTest, TestByteSlices, TestEarlyTermination, TestRollback, TestWildcard, TestDocCount, TestMmapDirectory, TestSpanExplanationsOfNonMatches, TestBagOfPositions, TestInPlaceMergeSorter, TestSparseFixedBitSet, TestAutomaton, Nested1, TestFilteredQuery, TestLucene50TermVectorsFormat, TestDocValuesTermsQuery, TestDocBoost, TestPhraseQuery, TestWindowsMMap, TestFastDecompressionMode, TestMultiLevelSkipList, TestCachingWrapperQuery, TestMatchAllDocsQuery, TestNamedSPILoader, TestBytesRefHash, TestPerFieldPostingsFormat2, TestForUtil, TestIndexWriterForceMerge, TestIndexableField, TestLock, TestMaxTermFrequency, TestIndexWriterOnJRECrash, TestLRUQueryCache]
>>>>>    [junit4] Completed on J3 in 25.65s, 12 tests, 1 failure <<< FAILURES!
>>>>>
>>>>> [...truncated 995 lines...]
>>>>> BUILD FAILED
>>>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
>>>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
>>>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
>>>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
>>>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:49: The following error occurred while executing this line:
>>>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
>>>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 419 suites, 3370 tests, 1 failure, 39 ignored (29 assumptions)
>>>>>
>>>>> Total time: 48 minutes 1 second
>>>>> Build step 'Invoke Ant' marked build as failure
>>>>> Archiving artifacts
>>>>> Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
>>>>> Archived 5 artifacts
>>>>> Archive block size is 32768
>>>>> Received 0 blocks and 3613290 bytes
>>>>> Compression is 0.0%
>>>>> Took 1 sec
>>>>> Recording test results
>>>>> Email was triggered for: Failure
>>>>> Sending email for trigger: Failure
>>>>>
>>>>>
>>>>>
>>>>>
>>>>> ---------------------------------------------------------------------
>>>>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>>>>> For additional commands, e-mail: dev-help@lucene.apache.org
>>>>
>>>>
>>>>
>>>> --
>>>> Adrien
>>>
>>>
>>>
>>> --
>>> Adrien
>>>
>>> ---------------------------------------------------------------------
>>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>>> For additional commands, e-mail: dev-help@lucene.apache.org
>>>
>>
>> ---------------------------------------------------------------------
>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>> For additional commands, e-mail: dev-help@lucene.apache.org
>>
>
>
>
> --
> Adrien
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


Re: [JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 775 - Still Failing

Posted by Adrien Grand <jp...@gmail.com>.
OK, it works for me too. Then I'll also add assertions that you do not
call score() if you passed needsScores=false.

On Tue, Mar 3, 2015 at 5:38 PM, Robert Muir <rc...@gmail.com> wrote:
> one option is to wrap with constantscorer somewhere and still keep the
> optimization?
>
> On Tue, Mar 3, 2015 at 11:32 AM, Adrien Grand <jp...@gmail.com> wrote:
>> OK, I found the reason. We have an optimization in disjunctions to
>> stop calling matches() after a single matching clause has been found
>> if scores are not needed, but it does not play well with the fact that
>> some scorers always call score() on the underlying clauses even if
>> scores are not needed (BooleanScorer in this case, maybe there are
>> other ones). So I'll disable the optimization. We'll need to better
>> document when it is legal to call score() if we want to bring it back.
>>
>> On Tue, Mar 3, 2015 at 3:57 PM, Adrien Grand <jp...@gmail.com> wrote:
>>> I'm looking into it.
>>>
>>> On Tue, Mar 3, 2015 at 3:52 PM, Apache Jenkins Server
>>> <je...@builds.apache.org> wrote:
>>>> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/775/
>>>>
>>>> 1 tests failed.
>>>> FAILED:  org.apache.lucene.search.TestLRUQueryCache.testRandom
>>>>
>>>> Error Message:
>>>>
>>>>
>>>> Stack Trace:
>>>> java.lang.AssertionError
>>>>         at __randomizedtesting.SeedInfo.seed([8AFB9C1E9D5CF8AF:F8B7B9112C3C4EDC]:0)
>>>>         at org.apache.lucene.search.TermScorer.score(TermScorer.java:70)
>>>>         at org.apache.lucene.search.DisjunctionSumScorer.score(DisjunctionSumScorer.java:46)
>>>>         at org.apache.lucene.search.DisjunctionScorer.score(DisjunctionScorer.java:219)
>>>>         at org.apache.lucene.search.ReqExclScorer.score(ReqExclScorer.java:121)
>>>>         at org.apache.lucene.search.BooleanScorer$OrCollector.collect(BooleanScorer.java:142)
>>>>         at org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:181)
>>>>         at org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:171)
>>>>         at org.apache.lucene.search.BooleanScorer$BulkScorerAndDoc.score(BooleanScorer.java:60)
>>>>         at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:210)
>>>>         at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:264)
>>>>         at org.apache.lucene.search.BooleanScorer.score(BooleanScorer.java:289)
>>>>         at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:79)
>>>>         at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:63)
>>>>         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:694)
>>>>         at org.apache.lucene.search.AssertingIndexSearcher.search(AssertingIndexSearcher.java:102)
>>>>         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:408)
>>>>         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:617)
>>>>         at org.apache.lucene.search.IndexSearcher.count(IndexSearcher.java:290)
>>>>         at org.apache.lucene.search.TestLRUQueryCache.testRandom(TestLRUQueryCache.java:923)
>>>>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>>>>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>>>>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>>>         at java.lang.reflect.Method.invoke(Method.java:606)
>>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
>>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
>>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
>>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
>>>>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>>>>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>>>>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>>>>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>>>>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>>>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>>>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
>>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
>>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
>>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
>>>>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>>>>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>>>>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>>>>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>>>>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>>>>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>>>>         at java.lang.Thread.run(Thread.java:745)
>>>>
>>>>
>>>>
>>>>
>>>> Build Log:
>>>> [...truncated 839 lines...]
>>>>    [junit4] Suite: org.apache.lucene.search.TestLRUQueryCache
>>>>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>>>>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLRUQueryCache -Dtests.method=testRandom -Dtests.seed=8AFB9C1E9D5CF8AF -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=en_ZA -Dtests.timezone=Africa/Porto-Novo -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
>>>>    [junit4] FAILURE 19.5s J3 | TestLRUQueryCache.testRandom <<<
>>>>    [junit4]    > Throwable #1: java.lang.AssertionError
>>>>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([8AFB9C1E9D5CF8AF:F8B7B9112C3C4EDC]:0)
>>>>    [junit4]    >        at org.apache.lucene.search.TermScorer.score(TermScorer.java:70)
>>>>    [junit4]    >        at org.apache.lucene.search.DisjunctionSumScorer.score(DisjunctionSumScorer.java:46)
>>>>    [junit4]    >        at org.apache.lucene.search.DisjunctionScorer.score(DisjunctionScorer.java:219)
>>>>    [junit4]    >        at org.apache.lucene.search.ReqExclScorer.score(ReqExclScorer.java:121)
>>>>    [junit4]    >        at org.apache.lucene.search.BooleanScorer$OrCollector.collect(BooleanScorer.java:142)
>>>>    [junit4]    >        at org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:181)
>>>>    [junit4]    >        at org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:171)
>>>>    [junit4]    >        at org.apache.lucene.search.BooleanScorer$BulkScorerAndDoc.score(BooleanScorer.java:60)
>>>>    [junit4]    >        at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:210)
>>>>    [junit4]    >        at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:264)
>>>>    [junit4]    >        at org.apache.lucene.search.BooleanScorer.score(BooleanScorer.java:289)
>>>>    [junit4]    >        at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:79)
>>>>    [junit4]    >        at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:63)
>>>>    [junit4]    >        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:694)
>>>>    [junit4]    >        at org.apache.lucene.search.AssertingIndexSearcher.search(AssertingIndexSearcher.java:102)
>>>>    [junit4]    >        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:408)
>>>>    [junit4]    >        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:617)
>>>>    [junit4]    >        at org.apache.lucene.search.IndexSearcher.count(IndexSearcher.java:290)
>>>>    [junit4]    >        at org.apache.lucene.search.TestLRUQueryCache.testRandom(TestLRUQueryCache.java:923)
>>>>    [junit4]    >        at java.lang.Thread.run(Thread.java:745)
>>>>    [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestLRUQueryCache 8AFB9C1E9D5CF8AF-001
>>>>    [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50), sim=DefaultSimilarity, locale=en_ZA, timezone=Africa/Porto-Novo
>>>>    [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=148303864,total=313524224
>>>>    [junit4]   2> NOTE: All tests run in this JVM: [FuzzyTermOnShortTermsTest, TestByteSlices, TestEarlyTermination, TestRollback, TestWildcard, TestDocCount, TestMmapDirectory, TestSpanExplanationsOfNonMatches, TestBagOfPositions, TestInPlaceMergeSorter, TestSparseFixedBitSet, TestAutomaton, Nested1, TestFilteredQuery, TestLucene50TermVectorsFormat, TestDocValuesTermsQuery, TestDocBoost, TestPhraseQuery, TestWindowsMMap, TestFastDecompressionMode, TestMultiLevelSkipList, TestCachingWrapperQuery, TestMatchAllDocsQuery, TestNamedSPILoader, TestBytesRefHash, TestPerFieldPostingsFormat2, TestForUtil, TestIndexWriterForceMerge, TestIndexableField, TestLock, TestMaxTermFrequency, TestIndexWriterOnJRECrash, TestLRUQueryCache]
>>>>    [junit4] Completed on J3 in 25.65s, 12 tests, 1 failure <<< FAILURES!
>>>>
>>>> [...truncated 995 lines...]
>>>> BUILD FAILED
>>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
>>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
>>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
>>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
>>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:49: The following error occurred while executing this line:
>>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
>>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 419 suites, 3370 tests, 1 failure, 39 ignored (29 assumptions)
>>>>
>>>> Total time: 48 minutes 1 second
>>>> Build step 'Invoke Ant' marked build as failure
>>>> Archiving artifacts
>>>> Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
>>>> Archived 5 artifacts
>>>> Archive block size is 32768
>>>> Received 0 blocks and 3613290 bytes
>>>> Compression is 0.0%
>>>> Took 1 sec
>>>> Recording test results
>>>> Email was triggered for: Failure
>>>> Sending email for trigger: Failure
>>>>
>>>>
>>>>
>>>>
>>>> ---------------------------------------------------------------------
>>>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>>>> For additional commands, e-mail: dev-help@lucene.apache.org
>>>
>>>
>>>
>>> --
>>> Adrien
>>
>>
>>
>> --
>> Adrien
>>
>> ---------------------------------------------------------------------
>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>> For additional commands, e-mail: dev-help@lucene.apache.org
>>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>



-- 
Adrien

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


Re: [JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 775 - Still Failing

Posted by Robert Muir <rc...@gmail.com>.
one option is to wrap with constantscorer somewhere and still keep the
optimization?

On Tue, Mar 3, 2015 at 11:32 AM, Adrien Grand <jp...@gmail.com> wrote:
> OK, I found the reason. We have an optimization in disjunctions to
> stop calling matches() after a single matching clause has been found
> if scores are not needed, but it does not play well with the fact that
> some scorers always call score() on the underlying clauses even if
> scores are not needed (BooleanScorer in this case, maybe there are
> other ones). So I'll disable the optimization. We'll need to better
> document when it is legal to call score() if we want to bring it back.
>
> On Tue, Mar 3, 2015 at 3:57 PM, Adrien Grand <jp...@gmail.com> wrote:
>> I'm looking into it.
>>
>> On Tue, Mar 3, 2015 at 3:52 PM, Apache Jenkins Server
>> <je...@builds.apache.org> wrote:
>>> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/775/
>>>
>>> 1 tests failed.
>>> FAILED:  org.apache.lucene.search.TestLRUQueryCache.testRandom
>>>
>>> Error Message:
>>>
>>>
>>> Stack Trace:
>>> java.lang.AssertionError
>>>         at __randomizedtesting.SeedInfo.seed([8AFB9C1E9D5CF8AF:F8B7B9112C3C4EDC]:0)
>>>         at org.apache.lucene.search.TermScorer.score(TermScorer.java:70)
>>>         at org.apache.lucene.search.DisjunctionSumScorer.score(DisjunctionSumScorer.java:46)
>>>         at org.apache.lucene.search.DisjunctionScorer.score(DisjunctionScorer.java:219)
>>>         at org.apache.lucene.search.ReqExclScorer.score(ReqExclScorer.java:121)
>>>         at org.apache.lucene.search.BooleanScorer$OrCollector.collect(BooleanScorer.java:142)
>>>         at org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:181)
>>>         at org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:171)
>>>         at org.apache.lucene.search.BooleanScorer$BulkScorerAndDoc.score(BooleanScorer.java:60)
>>>         at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:210)
>>>         at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:264)
>>>         at org.apache.lucene.search.BooleanScorer.score(BooleanScorer.java:289)
>>>         at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:79)
>>>         at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:63)
>>>         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:694)
>>>         at org.apache.lucene.search.AssertingIndexSearcher.search(AssertingIndexSearcher.java:102)
>>>         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:408)
>>>         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:617)
>>>         at org.apache.lucene.search.IndexSearcher.count(IndexSearcher.java:290)
>>>         at org.apache.lucene.search.TestLRUQueryCache.testRandom(TestLRUQueryCache.java:923)
>>>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>>>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>>>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>>         at java.lang.reflect.Method.invoke(Method.java:606)
>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
>>>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>>>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>>>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>>>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>>>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
>>>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>>>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>>>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>>>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>>>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>>>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>>>         at java.lang.Thread.run(Thread.java:745)
>>>
>>>
>>>
>>>
>>> Build Log:
>>> [...truncated 839 lines...]
>>>    [junit4] Suite: org.apache.lucene.search.TestLRUQueryCache
>>>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>>>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLRUQueryCache -Dtests.method=testRandom -Dtests.seed=8AFB9C1E9D5CF8AF -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=en_ZA -Dtests.timezone=Africa/Porto-Novo -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
>>>    [junit4] FAILURE 19.5s J3 | TestLRUQueryCache.testRandom <<<
>>>    [junit4]    > Throwable #1: java.lang.AssertionError
>>>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([8AFB9C1E9D5CF8AF:F8B7B9112C3C4EDC]:0)
>>>    [junit4]    >        at org.apache.lucene.search.TermScorer.score(TermScorer.java:70)
>>>    [junit4]    >        at org.apache.lucene.search.DisjunctionSumScorer.score(DisjunctionSumScorer.java:46)
>>>    [junit4]    >        at org.apache.lucene.search.DisjunctionScorer.score(DisjunctionScorer.java:219)
>>>    [junit4]    >        at org.apache.lucene.search.ReqExclScorer.score(ReqExclScorer.java:121)
>>>    [junit4]    >        at org.apache.lucene.search.BooleanScorer$OrCollector.collect(BooleanScorer.java:142)
>>>    [junit4]    >        at org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:181)
>>>    [junit4]    >        at org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:171)
>>>    [junit4]    >        at org.apache.lucene.search.BooleanScorer$BulkScorerAndDoc.score(BooleanScorer.java:60)
>>>    [junit4]    >        at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:210)
>>>    [junit4]    >        at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:264)
>>>    [junit4]    >        at org.apache.lucene.search.BooleanScorer.score(BooleanScorer.java:289)
>>>    [junit4]    >        at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:79)
>>>    [junit4]    >        at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:63)
>>>    [junit4]    >        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:694)
>>>    [junit4]    >        at org.apache.lucene.search.AssertingIndexSearcher.search(AssertingIndexSearcher.java:102)
>>>    [junit4]    >        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:408)
>>>    [junit4]    >        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:617)
>>>    [junit4]    >        at org.apache.lucene.search.IndexSearcher.count(IndexSearcher.java:290)
>>>    [junit4]    >        at org.apache.lucene.search.TestLRUQueryCache.testRandom(TestLRUQueryCache.java:923)
>>>    [junit4]    >        at java.lang.Thread.run(Thread.java:745)
>>>    [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestLRUQueryCache 8AFB9C1E9D5CF8AF-001
>>>    [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50), sim=DefaultSimilarity, locale=en_ZA, timezone=Africa/Porto-Novo
>>>    [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=148303864,total=313524224
>>>    [junit4]   2> NOTE: All tests run in this JVM: [FuzzyTermOnShortTermsTest, TestByteSlices, TestEarlyTermination, TestRollback, TestWildcard, TestDocCount, TestMmapDirectory, TestSpanExplanationsOfNonMatches, TestBagOfPositions, TestInPlaceMergeSorter, TestSparseFixedBitSet, TestAutomaton, Nested1, TestFilteredQuery, TestLucene50TermVectorsFormat, TestDocValuesTermsQuery, TestDocBoost, TestPhraseQuery, TestWindowsMMap, TestFastDecompressionMode, TestMultiLevelSkipList, TestCachingWrapperQuery, TestMatchAllDocsQuery, TestNamedSPILoader, TestBytesRefHash, TestPerFieldPostingsFormat2, TestForUtil, TestIndexWriterForceMerge, TestIndexableField, TestLock, TestMaxTermFrequency, TestIndexWriterOnJRECrash, TestLRUQueryCache]
>>>    [junit4] Completed on J3 in 25.65s, 12 tests, 1 failure <<< FAILURES!
>>>
>>> [...truncated 995 lines...]
>>> BUILD FAILED
>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:49: The following error occurred while executing this line:
>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 419 suites, 3370 tests, 1 failure, 39 ignored (29 assumptions)
>>>
>>> Total time: 48 minutes 1 second
>>> Build step 'Invoke Ant' marked build as failure
>>> Archiving artifacts
>>> Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
>>> Archived 5 artifacts
>>> Archive block size is 32768
>>> Received 0 blocks and 3613290 bytes
>>> Compression is 0.0%
>>> Took 1 sec
>>> Recording test results
>>> Email was triggered for: Failure
>>> Sending email for trigger: Failure
>>>
>>>
>>>
>>>
>>> ---------------------------------------------------------------------
>>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>>> For additional commands, e-mail: dev-help@lucene.apache.org
>>
>>
>>
>> --
>> Adrien
>
>
>
> --
> Adrien
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


Re: [JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 775 - Still Failing

Posted by Adrien Grand <jp...@gmail.com>.
OK, I found the reason. We have an optimization in disjunctions to
stop calling matches() after a single matching clause has been found
if scores are not needed, but it does not play well with the fact that
some scorers always call score() on the underlying clauses even if
scores are not needed (BooleanScorer in this case, maybe there are
other ones). So I'll disable the optimization. We'll need to better
document when it is legal to call score() if we want to bring it back.

On Tue, Mar 3, 2015 at 3:57 PM, Adrien Grand <jp...@gmail.com> wrote:
> I'm looking into it.
>
> On Tue, Mar 3, 2015 at 3:52 PM, Apache Jenkins Server
> <je...@builds.apache.org> wrote:
>> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/775/
>>
>> 1 tests failed.
>> FAILED:  org.apache.lucene.search.TestLRUQueryCache.testRandom
>>
>> Error Message:
>>
>>
>> Stack Trace:
>> java.lang.AssertionError
>>         at __randomizedtesting.SeedInfo.seed([8AFB9C1E9D5CF8AF:F8B7B9112C3C4EDC]:0)
>>         at org.apache.lucene.search.TermScorer.score(TermScorer.java:70)
>>         at org.apache.lucene.search.DisjunctionSumScorer.score(DisjunctionSumScorer.java:46)
>>         at org.apache.lucene.search.DisjunctionScorer.score(DisjunctionScorer.java:219)
>>         at org.apache.lucene.search.ReqExclScorer.score(ReqExclScorer.java:121)
>>         at org.apache.lucene.search.BooleanScorer$OrCollector.collect(BooleanScorer.java:142)
>>         at org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:181)
>>         at org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:171)
>>         at org.apache.lucene.search.BooleanScorer$BulkScorerAndDoc.score(BooleanScorer.java:60)
>>         at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:210)
>>         at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:264)
>>         at org.apache.lucene.search.BooleanScorer.score(BooleanScorer.java:289)
>>         at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:79)
>>         at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:63)
>>         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:694)
>>         at org.apache.lucene.search.AssertingIndexSearcher.search(AssertingIndexSearcher.java:102)
>>         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:408)
>>         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:617)
>>         at org.apache.lucene.search.IndexSearcher.count(IndexSearcher.java:290)
>>         at org.apache.lucene.search.TestLRUQueryCache.testRandom(TestLRUQueryCache.java:923)
>>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>         at java.lang.reflect.Method.invoke(Method.java:606)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
>>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
>>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>>         at java.lang.Thread.run(Thread.java:745)
>>
>>
>>
>>
>> Build Log:
>> [...truncated 839 lines...]
>>    [junit4] Suite: org.apache.lucene.search.TestLRUQueryCache
>>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLRUQueryCache -Dtests.method=testRandom -Dtests.seed=8AFB9C1E9D5CF8AF -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=en_ZA -Dtests.timezone=Africa/Porto-Novo -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
>>    [junit4] FAILURE 19.5s J3 | TestLRUQueryCache.testRandom <<<
>>    [junit4]    > Throwable #1: java.lang.AssertionError
>>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([8AFB9C1E9D5CF8AF:F8B7B9112C3C4EDC]:0)
>>    [junit4]    >        at org.apache.lucene.search.TermScorer.score(TermScorer.java:70)
>>    [junit4]    >        at org.apache.lucene.search.DisjunctionSumScorer.score(DisjunctionSumScorer.java:46)
>>    [junit4]    >        at org.apache.lucene.search.DisjunctionScorer.score(DisjunctionScorer.java:219)
>>    [junit4]    >        at org.apache.lucene.search.ReqExclScorer.score(ReqExclScorer.java:121)
>>    [junit4]    >        at org.apache.lucene.search.BooleanScorer$OrCollector.collect(BooleanScorer.java:142)
>>    [junit4]    >        at org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:181)
>>    [junit4]    >        at org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:171)
>>    [junit4]    >        at org.apache.lucene.search.BooleanScorer$BulkScorerAndDoc.score(BooleanScorer.java:60)
>>    [junit4]    >        at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:210)
>>    [junit4]    >        at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:264)
>>    [junit4]    >        at org.apache.lucene.search.BooleanScorer.score(BooleanScorer.java:289)
>>    [junit4]    >        at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:79)
>>    [junit4]    >        at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:63)
>>    [junit4]    >        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:694)
>>    [junit4]    >        at org.apache.lucene.search.AssertingIndexSearcher.search(AssertingIndexSearcher.java:102)
>>    [junit4]    >        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:408)
>>    [junit4]    >        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:617)
>>    [junit4]    >        at org.apache.lucene.search.IndexSearcher.count(IndexSearcher.java:290)
>>    [junit4]    >        at org.apache.lucene.search.TestLRUQueryCache.testRandom(TestLRUQueryCache.java:923)
>>    [junit4]    >        at java.lang.Thread.run(Thread.java:745)
>>    [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestLRUQueryCache 8AFB9C1E9D5CF8AF-001
>>    [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50), sim=DefaultSimilarity, locale=en_ZA, timezone=Africa/Porto-Novo
>>    [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=148303864,total=313524224
>>    [junit4]   2> NOTE: All tests run in this JVM: [FuzzyTermOnShortTermsTest, TestByteSlices, TestEarlyTermination, TestRollback, TestWildcard, TestDocCount, TestMmapDirectory, TestSpanExplanationsOfNonMatches, TestBagOfPositions, TestInPlaceMergeSorter, TestSparseFixedBitSet, TestAutomaton, Nested1, TestFilteredQuery, TestLucene50TermVectorsFormat, TestDocValuesTermsQuery, TestDocBoost, TestPhraseQuery, TestWindowsMMap, TestFastDecompressionMode, TestMultiLevelSkipList, TestCachingWrapperQuery, TestMatchAllDocsQuery, TestNamedSPILoader, TestBytesRefHash, TestPerFieldPostingsFormat2, TestForUtil, TestIndexWriterForceMerge, TestIndexableField, TestLock, TestMaxTermFrequency, TestIndexWriterOnJRECrash, TestLRUQueryCache]
>>    [junit4] Completed on J3 in 25.65s, 12 tests, 1 failure <<< FAILURES!
>>
>> [...truncated 995 lines...]
>> BUILD FAILED
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:49: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 419 suites, 3370 tests, 1 failure, 39 ignored (29 assumptions)
>>
>> Total time: 48 minutes 1 second
>> Build step 'Invoke Ant' marked build as failure
>> Archiving artifacts
>> Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
>> Archived 5 artifacts
>> Archive block size is 32768
>> Received 0 blocks and 3613290 bytes
>> Compression is 0.0%
>> Took 1 sec
>> Recording test results
>> Email was triggered for: Failure
>> Sending email for trigger: Failure
>>
>>
>>
>>
>> ---------------------------------------------------------------------
>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>> For additional commands, e-mail: dev-help@lucene.apache.org
>
>
>
> --
> Adrien



-- 
Adrien

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


Re: [JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 775 - Still Failing

Posted by Adrien Grand <jp...@gmail.com>.
I'm looking into it.

On Tue, Mar 3, 2015 at 3:52 PM, Apache Jenkins Server
<je...@builds.apache.org> wrote:
> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/775/
>
> 1 tests failed.
> FAILED:  org.apache.lucene.search.TestLRUQueryCache.testRandom
>
> Error Message:
>
>
> Stack Trace:
> java.lang.AssertionError
>         at __randomizedtesting.SeedInfo.seed([8AFB9C1E9D5CF8AF:F8B7B9112C3C4EDC]:0)
>         at org.apache.lucene.search.TermScorer.score(TermScorer.java:70)
>         at org.apache.lucene.search.DisjunctionSumScorer.score(DisjunctionSumScorer.java:46)
>         at org.apache.lucene.search.DisjunctionScorer.score(DisjunctionScorer.java:219)
>         at org.apache.lucene.search.ReqExclScorer.score(ReqExclScorer.java:121)
>         at org.apache.lucene.search.BooleanScorer$OrCollector.collect(BooleanScorer.java:142)
>         at org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:181)
>         at org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:171)
>         at org.apache.lucene.search.BooleanScorer$BulkScorerAndDoc.score(BooleanScorer.java:60)
>         at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:210)
>         at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:264)
>         at org.apache.lucene.search.BooleanScorer.score(BooleanScorer.java:289)
>         at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:79)
>         at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:63)
>         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:694)
>         at org.apache.lucene.search.AssertingIndexSearcher.search(AssertingIndexSearcher.java:102)
>         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:408)
>         at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:617)
>         at org.apache.lucene.search.IndexSearcher.count(IndexSearcher.java:290)
>         at org.apache.lucene.search.TestLRUQueryCache.testRandom(TestLRUQueryCache.java:923)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
>
>
> Build Log:
> [...truncated 839 lines...]
>    [junit4] Suite: org.apache.lucene.search.TestLRUQueryCache
>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLRUQueryCache -Dtests.method=testRandom -Dtests.seed=8AFB9C1E9D5CF8AF -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=en_ZA -Dtests.timezone=Africa/Porto-Novo -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
>    [junit4] FAILURE 19.5s J3 | TestLRUQueryCache.testRandom <<<
>    [junit4]    > Throwable #1: java.lang.AssertionError
>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([8AFB9C1E9D5CF8AF:F8B7B9112C3C4EDC]:0)
>    [junit4]    >        at org.apache.lucene.search.TermScorer.score(TermScorer.java:70)
>    [junit4]    >        at org.apache.lucene.search.DisjunctionSumScorer.score(DisjunctionSumScorer.java:46)
>    [junit4]    >        at org.apache.lucene.search.DisjunctionScorer.score(DisjunctionScorer.java:219)
>    [junit4]    >        at org.apache.lucene.search.ReqExclScorer.score(ReqExclScorer.java:121)
>    [junit4]    >        at org.apache.lucene.search.BooleanScorer$OrCollector.collect(BooleanScorer.java:142)
>    [junit4]    >        at org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:181)
>    [junit4]    >        at org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:171)
>    [junit4]    >        at org.apache.lucene.search.BooleanScorer$BulkScorerAndDoc.score(BooleanScorer.java:60)
>    [junit4]    >        at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:210)
>    [junit4]    >        at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:264)
>    [junit4]    >        at org.apache.lucene.search.BooleanScorer.score(BooleanScorer.java:289)
>    [junit4]    >        at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:79)
>    [junit4]    >        at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:63)
>    [junit4]    >        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:694)
>    [junit4]    >        at org.apache.lucene.search.AssertingIndexSearcher.search(AssertingIndexSearcher.java:102)
>    [junit4]    >        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:408)
>    [junit4]    >        at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:617)
>    [junit4]    >        at org.apache.lucene.search.IndexSearcher.count(IndexSearcher.java:290)
>    [junit4]    >        at org.apache.lucene.search.TestLRUQueryCache.testRandom(TestLRUQueryCache.java:923)
>    [junit4]    >        at java.lang.Thread.run(Thread.java:745)
>    [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestLRUQueryCache 8AFB9C1E9D5CF8AF-001
>    [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50), sim=DefaultSimilarity, locale=en_ZA, timezone=Africa/Porto-Novo
>    [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=148303864,total=313524224
>    [junit4]   2> NOTE: All tests run in this JVM: [FuzzyTermOnShortTermsTest, TestByteSlices, TestEarlyTermination, TestRollback, TestWildcard, TestDocCount, TestMmapDirectory, TestSpanExplanationsOfNonMatches, TestBagOfPositions, TestInPlaceMergeSorter, TestSparseFixedBitSet, TestAutomaton, Nested1, TestFilteredQuery, TestLucene50TermVectorsFormat, TestDocValuesTermsQuery, TestDocBoost, TestPhraseQuery, TestWindowsMMap, TestFastDecompressionMode, TestMultiLevelSkipList, TestCachingWrapperQuery, TestMatchAllDocsQuery, TestNamedSPILoader, TestBytesRefHash, TestPerFieldPostingsFormat2, TestForUtil, TestIndexWriterForceMerge, TestIndexableField, TestLock, TestMaxTermFrequency, TestIndexWriterOnJRECrash, TestLRUQueryCache]
>    [junit4] Completed on J3 in 25.65s, 12 tests, 1 failure <<< FAILURES!
>
> [...truncated 995 lines...]
> BUILD FAILED
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:49: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 419 suites, 3370 tests, 1 failure, 39 ignored (29 assumptions)
>
> Total time: 48 minutes 1 second
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
> Archived 5 artifacts
> Archive block size is 32768
> Received 0 blocks and 3613290 bytes
> Compression is 0.0%
> Took 1 sec
> Recording test results
> Email was triggered for: Failure
> Sending email for trigger: Failure
>
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org



-- 
Adrien

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 775 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/775/

1 tests failed.
FAILED:  org.apache.lucene.search.TestLRUQueryCache.testRandom

Error Message:


Stack Trace:
java.lang.AssertionError
	at __randomizedtesting.SeedInfo.seed([8AFB9C1E9D5CF8AF:F8B7B9112C3C4EDC]:0)
	at org.apache.lucene.search.TermScorer.score(TermScorer.java:70)
	at org.apache.lucene.search.DisjunctionSumScorer.score(DisjunctionSumScorer.java:46)
	at org.apache.lucene.search.DisjunctionScorer.score(DisjunctionScorer.java:219)
	at org.apache.lucene.search.ReqExclScorer.score(ReqExclScorer.java:121)
	at org.apache.lucene.search.BooleanScorer$OrCollector.collect(BooleanScorer.java:142)
	at org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:181)
	at org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:171)
	at org.apache.lucene.search.BooleanScorer$BulkScorerAndDoc.score(BooleanScorer.java:60)
	at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:210)
	at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:264)
	at org.apache.lucene.search.BooleanScorer.score(BooleanScorer.java:289)
	at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:79)
	at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:63)
	at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:694)
	at org.apache.lucene.search.AssertingIndexSearcher.search(AssertingIndexSearcher.java:102)
	at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:408)
	at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:617)
	at org.apache.lucene.search.IndexSearcher.count(IndexSearcher.java:290)
	at org.apache.lucene.search.TestLRUQueryCache.testRandom(TestLRUQueryCache.java:923)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1627)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:872)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:886)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:845)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:747)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:781)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:792)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 839 lines...]
   [junit4] Suite: org.apache.lucene.search.TestLRUQueryCache
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestLRUQueryCache -Dtests.method=testRandom -Dtests.seed=8AFB9C1E9D5CF8AF -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=en_ZA -Dtests.timezone=Africa/Porto-Novo -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] FAILURE 19.5s J3 | TestLRUQueryCache.testRandom <<<
   [junit4]    > Throwable #1: java.lang.AssertionError
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([8AFB9C1E9D5CF8AF:F8B7B9112C3C4EDC]:0)
   [junit4]    > 	at org.apache.lucene.search.TermScorer.score(TermScorer.java:70)
   [junit4]    > 	at org.apache.lucene.search.DisjunctionSumScorer.score(DisjunctionSumScorer.java:46)
   [junit4]    > 	at org.apache.lucene.search.DisjunctionScorer.score(DisjunctionScorer.java:219)
   [junit4]    > 	at org.apache.lucene.search.ReqExclScorer.score(ReqExclScorer.java:121)
   [junit4]    > 	at org.apache.lucene.search.BooleanScorer$OrCollector.collect(BooleanScorer.java:142)
   [junit4]    > 	at org.apache.lucene.search.Weight$DefaultBulkScorer.scoreRange(Weight.java:181)
   [junit4]    > 	at org.apache.lucene.search.Weight$DefaultBulkScorer.score(Weight.java:171)
   [junit4]    > 	at org.apache.lucene.search.BooleanScorer$BulkScorerAndDoc.score(BooleanScorer.java:60)
   [junit4]    > 	at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:210)
   [junit4]    > 	at org.apache.lucene.search.BooleanScorer.scoreWindow(BooleanScorer.java:264)
   [junit4]    > 	at org.apache.lucene.search.BooleanScorer.score(BooleanScorer.java:289)
   [junit4]    > 	at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:79)
   [junit4]    > 	at org.apache.lucene.search.AssertingBulkScorer.score(AssertingBulkScorer.java:63)
   [junit4]    > 	at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:694)
   [junit4]    > 	at org.apache.lucene.search.AssertingIndexSearcher.search(AssertingIndexSearcher.java:102)
   [junit4]    > 	at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:408)
   [junit4]    > 	at org.apache.lucene.search.IndexSearcher.search(IndexSearcher.java:617)
   [junit4]    > 	at org.apache.lucene.search.IndexSearcher.count(IndexSearcher.java:290)
   [junit4]    > 	at org.apache.lucene.search.TestLRUQueryCache.testRandom(TestLRUQueryCache.java:923)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J3/temp/lucene.search.TestLRUQueryCache 8AFB9C1E9D5CF8AF-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50), sim=DefaultSimilarity, locale=en_ZA, timezone=Africa/Porto-Novo
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=148303864,total=313524224
   [junit4]   2> NOTE: All tests run in this JVM: [FuzzyTermOnShortTermsTest, TestByteSlices, TestEarlyTermination, TestRollback, TestWildcard, TestDocCount, TestMmapDirectory, TestSpanExplanationsOfNonMatches, TestBagOfPositions, TestInPlaceMergeSorter, TestSparseFixedBitSet, TestAutomaton, Nested1, TestFilteredQuery, TestLucene50TermVectorsFormat, TestDocValuesTermsQuery, TestDocBoost, TestPhraseQuery, TestWindowsMMap, TestFastDecompressionMode, TestMultiLevelSkipList, TestCachingWrapperQuery, TestMatchAllDocsQuery, TestNamedSPILoader, TestBytesRefHash, TestPerFieldPostingsFormat2, TestForUtil, TestIndexWriterForceMerge, TestIndexableField, TestLock, TestMaxTermFrequency, TestIndexWriterOnJRECrash, TestLRUQueryCache]
   [junit4] Completed on J3 in 25.65s, 12 tests, 1 failure <<< FAILURES!

[...truncated 995 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:49: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 419 suites, 3370 tests, 1 failure, 39 ignored (29 assumptions)

Total time: 48 minutes 1 second
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 3613290 bytes
Compression is 0.0%
Took 1 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 774 - Still Failing

Posted by Robert Muir <rc...@gmail.com>.
I opened https://issues.apache.org/jira/browse/LUCENE-6383 as a
followup. The fix should work for this test at least, i beasted the
test over 100 times and it seems ok now.

On Wed, Apr 1, 2015 at 12:02 AM, Robert Muir <rc...@gmail.com> wrote:
> I committed a fix. But I think memoryPF can get really wasteful here.
> This is ultimately the same problem as the TestDuelingCodecs OOM.
>
>
> On Tue, Mar 31, 2015 at 11:08 PM, Robert Muir <rc...@gmail.com> wrote:
>> This reproduces. I'm digging.
>>
>> On Mon, Mar 2, 2015 at 9:52 AM, Apache Jenkins Server
>> <je...@builds.apache.org> wrote:
>>> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/774/
>>>
>>> 1 tests failed.
>>> REGRESSION:  org.apache.lucene.index.TestIndexWriterForceMerge.testForceMergeTempSpaceUsage
>>>
>>> Error Message:
>>> forceMerge used too much temporary space: starting usage was 379542 bytes; final usage was 442916 bytes; max temp usage was 1669519 but should have been 1328748 (= 3X starting usage)
>>>
>>> Stack Trace:
>>> java.lang.AssertionError: forceMerge used too much temporary space: starting usage was 379542 bytes; final usage was 442916 bytes; max temp usage was 1669519 but should have been 1328748 (= 3X starting usage)
>>>         at __randomizedtesting.SeedInfo.seed([AD6008DD6F02F612:B7A2CB2E011215D5]:0)
>>>         at org.junit.Assert.fail(Assert.java:93)
>>>         at org.junit.Assert.assertTrue(Assert.java:43)
>>>         at org.apache.lucene.index.TestIndexWriterForceMerge.testForceMergeTempSpaceUsage(TestIndexWriterForceMerge.java:181)
>>>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>>>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>>>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>>         at java.lang.reflect.Method.invoke(Method.java:606)
>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>>>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>>>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>>>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>>>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>>>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>>>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>>>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>>>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>>>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>>>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>>>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>>>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>>>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>>>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>>>         at java.lang.Thread.run(Thread.java:745)
>>>
>>>
>>>
>>>
>>> Build Log:
>>> [...truncated 999 lines...]
>>>    [junit4] Suite: org.apache.lucene.index.TestIndexWriterForceMerge
>>>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>>>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestIndexWriterForceMerge -Dtests.method=testForceMergeTempSpaceUsage -Dtests.seed=AD6008DD6F02F612 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=en_GB -Dtests.timezone=Pacific/Midway -Dtests.asserts=true -Dtests.file.encoding=UTF-8
>>>    [junit4] FAILURE 2.11s J1 | TestIndexWriterForceMerge.testForceMergeTempSpaceUsage <<<
>>>    [junit4]    > Throwable #1: java.lang.AssertionError: forceMerge used too much temporary space: starting usage was 379542 bytes; final usage was 442916 bytes; max temp usage was 1669519 but should have been 1328748 (= 3X starting usage)
>>>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([AD6008DD6F02F612:B7A2CB2E011215D5]:0)
>>>    [junit4]    >        at org.apache.lucene.index.TestIndexWriterForceMerge.testForceMergeTempSpaceUsage(TestIndexWriterForceMerge.java:181)
>>>    [junit4]    >        at java.lang.Thread.run(Thread.java:745)
>>>    [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J1/temp/lucene.index.TestIndexWriterForceMerge AD6008DD6F02F612-001
>>>    [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {field=PostingsFormat(name=LuceneFixedGap), id=Lucene50(blocksize=128), content=PostingsFormat(name=Memory doPackFST= true)}, docValues:{}, sim=DefaultSimilarity, locale=en_GB, timezone=Pacific/Midway
>>>    [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=186424664,total=358088704
>>>    [junit4]   2> NOTE: All tests run in this JVM: [TestDocInverterPerFieldErrorInfo, TestStressIndexing, TestNumericUtils, TestLockFactory, TestFilteredSearch, TestDocCount, TestIsCurrent, TestConsistentFieldNumbers, TestSimpleFSDirectory, TestAssertingStoredFieldsFormat, Nested, Nested, TestParallelReaderEmptyIndex, TestDateFilter, TestQueryBuilder, TestCheckIndex, TestBufferedChecksum, TestTermVectors, TestIndexFileDeleter, TestLiveFieldValues, TestIndexWriterDelete, TestTopDocsCollector, TestMultiMMap, TestDocValuesScoring, TestFuzzyQuery, TestLucene50StoredFieldsFormat, TestStressAdvance, TestIndexSearcher, TestPerFieldDocValuesFormat, TestNumericTokenStream, TestPackedTokenAttributeImpl, TestManyFields, TestNumericDocValuesUpdates, TestGraphTokenizers, Test2BPagedBytes, TestTimSorter, TestEliasFanoDocIdSet, TestInfoStream, TestBlockPostingsFormat3, TestRegexpQuery, TestDocValuesRangeQuery, TestDocTermOrdsRewriteMethod, TestSloppyPhraseQuery2, TestMockDirectoryWrapper, TestBlockPostingsFormat, TestCharsRefBuilder, TestIndexWriterMergePolicy, TestBooleanQuery, TestMinShouldMatch2, TestTerm, TestIntsRef, TestAssertions, TestFieldMaskingSpanQuery, TestPostingsOffsets, TestArrayUtil, TestLongPostings, TestIndexWriterForceMerge]
>>>    [junit4] Completed on J1 in 6.51s, 4 tests, 1 failure <<< FAILURES!
>>>
>>> [...truncated 663 lines...]
>>> BUILD FAILED
>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:49: The following error occurred while executing this line:
>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
>>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 421 suites, 3373 tests, 1 failure, 40 ignored (30 assumptions)
>>>
>>> Total time: 48 minutes 55 seconds
>>> Build step 'Invoke Ant' marked build as failure
>>> Archiving artifacts
>>> Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
>>> Archived 5 artifacts
>>> Archive block size is 32768
>>> Received 0 blocks and 3647849 bytes
>>> Compression is 0.0%
>>> Took 1.1 sec
>>> Recording test results
>>> Email was triggered for: Failure
>>> Sending email for trigger: Failure
>>>
>>>
>>>
>>>
>>> ---------------------------------------------------------------------
>>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>>> For additional commands, e-mail: dev-help@lucene.apache.org

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


Re: [JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 774 - Still Failing

Posted by Robert Muir <rc...@gmail.com>.
I committed a fix. But I think memoryPF can get really wasteful here.
This is ultimately the same problem as the TestDuelingCodecs OOM.


On Tue, Mar 31, 2015 at 11:08 PM, Robert Muir <rc...@gmail.com> wrote:
> This reproduces. I'm digging.
>
> On Mon, Mar 2, 2015 at 9:52 AM, Apache Jenkins Server
> <je...@builds.apache.org> wrote:
>> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/774/
>>
>> 1 tests failed.
>> REGRESSION:  org.apache.lucene.index.TestIndexWriterForceMerge.testForceMergeTempSpaceUsage
>>
>> Error Message:
>> forceMerge used too much temporary space: starting usage was 379542 bytes; final usage was 442916 bytes; max temp usage was 1669519 but should have been 1328748 (= 3X starting usage)
>>
>> Stack Trace:
>> java.lang.AssertionError: forceMerge used too much temporary space: starting usage was 379542 bytes; final usage was 442916 bytes; max temp usage was 1669519 but should have been 1328748 (= 3X starting usage)
>>         at __randomizedtesting.SeedInfo.seed([AD6008DD6F02F612:B7A2CB2E011215D5]:0)
>>         at org.junit.Assert.fail(Assert.java:93)
>>         at org.junit.Assert.assertTrue(Assert.java:43)
>>         at org.apache.lucene.index.TestIndexWriterForceMerge.testForceMergeTempSpaceUsage(TestIndexWriterForceMerge.java:181)
>>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>         at java.lang.reflect.Method.invoke(Method.java:606)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>>         at java.lang.Thread.run(Thread.java:745)
>>
>>
>>
>>
>> Build Log:
>> [...truncated 999 lines...]
>>    [junit4] Suite: org.apache.lucene.index.TestIndexWriterForceMerge
>>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestIndexWriterForceMerge -Dtests.method=testForceMergeTempSpaceUsage -Dtests.seed=AD6008DD6F02F612 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=en_GB -Dtests.timezone=Pacific/Midway -Dtests.asserts=true -Dtests.file.encoding=UTF-8
>>    [junit4] FAILURE 2.11s J1 | TestIndexWriterForceMerge.testForceMergeTempSpaceUsage <<<
>>    [junit4]    > Throwable #1: java.lang.AssertionError: forceMerge used too much temporary space: starting usage was 379542 bytes; final usage was 442916 bytes; max temp usage was 1669519 but should have been 1328748 (= 3X starting usage)
>>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([AD6008DD6F02F612:B7A2CB2E011215D5]:0)
>>    [junit4]    >        at org.apache.lucene.index.TestIndexWriterForceMerge.testForceMergeTempSpaceUsage(TestIndexWriterForceMerge.java:181)
>>    [junit4]    >        at java.lang.Thread.run(Thread.java:745)
>>    [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J1/temp/lucene.index.TestIndexWriterForceMerge AD6008DD6F02F612-001
>>    [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {field=PostingsFormat(name=LuceneFixedGap), id=Lucene50(blocksize=128), content=PostingsFormat(name=Memory doPackFST= true)}, docValues:{}, sim=DefaultSimilarity, locale=en_GB, timezone=Pacific/Midway
>>    [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=186424664,total=358088704
>>    [junit4]   2> NOTE: All tests run in this JVM: [TestDocInverterPerFieldErrorInfo, TestStressIndexing, TestNumericUtils, TestLockFactory, TestFilteredSearch, TestDocCount, TestIsCurrent, TestConsistentFieldNumbers, TestSimpleFSDirectory, TestAssertingStoredFieldsFormat, Nested, Nested, TestParallelReaderEmptyIndex, TestDateFilter, TestQueryBuilder, TestCheckIndex, TestBufferedChecksum, TestTermVectors, TestIndexFileDeleter, TestLiveFieldValues, TestIndexWriterDelete, TestTopDocsCollector, TestMultiMMap, TestDocValuesScoring, TestFuzzyQuery, TestLucene50StoredFieldsFormat, TestStressAdvance, TestIndexSearcher, TestPerFieldDocValuesFormat, TestNumericTokenStream, TestPackedTokenAttributeImpl, TestManyFields, TestNumericDocValuesUpdates, TestGraphTokenizers, Test2BPagedBytes, TestTimSorter, TestEliasFanoDocIdSet, TestInfoStream, TestBlockPostingsFormat3, TestRegexpQuery, TestDocValuesRangeQuery, TestDocTermOrdsRewriteMethod, TestSloppyPhraseQuery2, TestMockDirectoryWrapper, TestBlockPostingsFormat, TestCharsRefBuilder, TestIndexWriterMergePolicy, TestBooleanQuery, TestMinShouldMatch2, TestTerm, TestIntsRef, TestAssertions, TestFieldMaskingSpanQuery, TestPostingsOffsets, TestArrayUtil, TestLongPostings, TestIndexWriterForceMerge]
>>    [junit4] Completed on J1 in 6.51s, 4 tests, 1 failure <<< FAILURES!
>>
>> [...truncated 663 lines...]
>> BUILD FAILED
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:49: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
>> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 421 suites, 3373 tests, 1 failure, 40 ignored (30 assumptions)
>>
>> Total time: 48 minutes 55 seconds
>> Build step 'Invoke Ant' marked build as failure
>> Archiving artifacts
>> Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
>> Archived 5 artifacts
>> Archive block size is 32768
>> Received 0 blocks and 3647849 bytes
>> Compression is 0.0%
>> Took 1.1 sec
>> Recording test results
>> Email was triggered for: Failure
>> Sending email for trigger: Failure
>>
>>
>>
>>
>> ---------------------------------------------------------------------
>> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
>> For additional commands, e-mail: dev-help@lucene.apache.org

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


Re: [JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 774 - Still Failing

Posted by Robert Muir <rc...@gmail.com>.
This reproduces. I'm digging.

On Mon, Mar 2, 2015 at 9:52 AM, Apache Jenkins Server
<je...@builds.apache.org> wrote:
> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/774/
>
> 1 tests failed.
> REGRESSION:  org.apache.lucene.index.TestIndexWriterForceMerge.testForceMergeTempSpaceUsage
>
> Error Message:
> forceMerge used too much temporary space: starting usage was 379542 bytes; final usage was 442916 bytes; max temp usage was 1669519 but should have been 1328748 (= 3X starting usage)
>
> Stack Trace:
> java.lang.AssertionError: forceMerge used too much temporary space: starting usage was 379542 bytes; final usage was 442916 bytes; max temp usage was 1669519 but should have been 1328748 (= 3X starting usage)
>         at __randomizedtesting.SeedInfo.seed([AD6008DD6F02F612:B7A2CB2E011215D5]:0)
>         at org.junit.Assert.fail(Assert.java:93)
>         at org.junit.Assert.assertTrue(Assert.java:43)
>         at org.apache.lucene.index.TestIndexWriterForceMerge.testForceMergeTempSpaceUsage(TestIndexWriterForceMerge.java:181)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
>
>
> Build Log:
> [...truncated 999 lines...]
>    [junit4] Suite: org.apache.lucene.index.TestIndexWriterForceMerge
>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestIndexWriterForceMerge -Dtests.method=testForceMergeTempSpaceUsage -Dtests.seed=AD6008DD6F02F612 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=en_GB -Dtests.timezone=Pacific/Midway -Dtests.asserts=true -Dtests.file.encoding=UTF-8
>    [junit4] FAILURE 2.11s J1 | TestIndexWriterForceMerge.testForceMergeTempSpaceUsage <<<
>    [junit4]    > Throwable #1: java.lang.AssertionError: forceMerge used too much temporary space: starting usage was 379542 bytes; final usage was 442916 bytes; max temp usage was 1669519 but should have been 1328748 (= 3X starting usage)
>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([AD6008DD6F02F612:B7A2CB2E011215D5]:0)
>    [junit4]    >        at org.apache.lucene.index.TestIndexWriterForceMerge.testForceMergeTempSpaceUsage(TestIndexWriterForceMerge.java:181)
>    [junit4]    >        at java.lang.Thread.run(Thread.java:745)
>    [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J1/temp/lucene.index.TestIndexWriterForceMerge AD6008DD6F02F612-001
>    [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {field=PostingsFormat(name=LuceneFixedGap), id=Lucene50(blocksize=128), content=PostingsFormat(name=Memory doPackFST= true)}, docValues:{}, sim=DefaultSimilarity, locale=en_GB, timezone=Pacific/Midway
>    [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=186424664,total=358088704
>    [junit4]   2> NOTE: All tests run in this JVM: [TestDocInverterPerFieldErrorInfo, TestStressIndexing, TestNumericUtils, TestLockFactory, TestFilteredSearch, TestDocCount, TestIsCurrent, TestConsistentFieldNumbers, TestSimpleFSDirectory, TestAssertingStoredFieldsFormat, Nested, Nested, TestParallelReaderEmptyIndex, TestDateFilter, TestQueryBuilder, TestCheckIndex, TestBufferedChecksum, TestTermVectors, TestIndexFileDeleter, TestLiveFieldValues, TestIndexWriterDelete, TestTopDocsCollector, TestMultiMMap, TestDocValuesScoring, TestFuzzyQuery, TestLucene50StoredFieldsFormat, TestStressAdvance, TestIndexSearcher, TestPerFieldDocValuesFormat, TestNumericTokenStream, TestPackedTokenAttributeImpl, TestManyFields, TestNumericDocValuesUpdates, TestGraphTokenizers, Test2BPagedBytes, TestTimSorter, TestEliasFanoDocIdSet, TestInfoStream, TestBlockPostingsFormat3, TestRegexpQuery, TestDocValuesRangeQuery, TestDocTermOrdsRewriteMethod, TestSloppyPhraseQuery2, TestMockDirectoryWrapper, TestBlockPostingsFormat, TestCharsRefBuilder, TestIndexWriterMergePolicy, TestBooleanQuery, TestMinShouldMatch2, TestTerm, TestIntsRef, TestAssertions, TestFieldMaskingSpanQuery, TestPostingsOffsets, TestArrayUtil, TestLongPostings, TestIndexWriterForceMerge]
>    [junit4] Completed on J1 in 6.51s, 4 tests, 1 failure <<< FAILURES!
>
> [...truncated 663 lines...]
> BUILD FAILED
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:49: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 421 suites, 3373 tests, 1 failure, 40 ignored (30 assumptions)
>
> Total time: 48 minutes 55 seconds
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
> Archived 5 artifacts
> Archive block size is 32768
> Received 0 blocks and 3647849 bytes
> Compression is 0.0%
> Took 1.1 sec
> Recording test results
> Email was triggered for: Failure
> Sending email for trigger: Failure
>
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 774 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/774/

1 tests failed.
REGRESSION:  org.apache.lucene.index.TestIndexWriterForceMerge.testForceMergeTempSpaceUsage

Error Message:
forceMerge used too much temporary space: starting usage was 379542 bytes; final usage was 442916 bytes; max temp usage was 1669519 but should have been 1328748 (= 3X starting usage)

Stack Trace:
java.lang.AssertionError: forceMerge used too much temporary space: starting usage was 379542 bytes; final usage was 442916 bytes; max temp usage was 1669519 but should have been 1328748 (= 3X starting usage)
	at __randomizedtesting.SeedInfo.seed([AD6008DD6F02F612:B7A2CB2E011215D5]:0)
	at org.junit.Assert.fail(Assert.java:93)
	at org.junit.Assert.assertTrue(Assert.java:43)
	at org.apache.lucene.index.TestIndexWriterForceMerge.testForceMergeTempSpaceUsage(TestIndexWriterForceMerge.java:181)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 999 lines...]
   [junit4] Suite: org.apache.lucene.index.TestIndexWriterForceMerge
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestIndexWriterForceMerge -Dtests.method=testForceMergeTempSpaceUsage -Dtests.seed=AD6008DD6F02F612 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=en_GB -Dtests.timezone=Pacific/Midway -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] FAILURE 2.11s J1 | TestIndexWriterForceMerge.testForceMergeTempSpaceUsage <<<
   [junit4]    > Throwable #1: java.lang.AssertionError: forceMerge used too much temporary space: starting usage was 379542 bytes; final usage was 442916 bytes; max temp usage was 1669519 but should have been 1328748 (= 3X starting usage)
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([AD6008DD6F02F612:B7A2CB2E011215D5]:0)
   [junit4]    > 	at org.apache.lucene.index.TestIndexWriterForceMerge.testForceMergeTempSpaceUsage(TestIndexWriterForceMerge.java:181)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J1/temp/lucene.index.TestIndexWriterForceMerge AD6008DD6F02F612-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {field=PostingsFormat(name=LuceneFixedGap), id=Lucene50(blocksize=128), content=PostingsFormat(name=Memory doPackFST= true)}, docValues:{}, sim=DefaultSimilarity, locale=en_GB, timezone=Pacific/Midway
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=186424664,total=358088704
   [junit4]   2> NOTE: All tests run in this JVM: [TestDocInverterPerFieldErrorInfo, TestStressIndexing, TestNumericUtils, TestLockFactory, TestFilteredSearch, TestDocCount, TestIsCurrent, TestConsistentFieldNumbers, TestSimpleFSDirectory, TestAssertingStoredFieldsFormat, Nested, Nested, TestParallelReaderEmptyIndex, TestDateFilter, TestQueryBuilder, TestCheckIndex, TestBufferedChecksum, TestTermVectors, TestIndexFileDeleter, TestLiveFieldValues, TestIndexWriterDelete, TestTopDocsCollector, TestMultiMMap, TestDocValuesScoring, TestFuzzyQuery, TestLucene50StoredFieldsFormat, TestStressAdvance, TestIndexSearcher, TestPerFieldDocValuesFormat, TestNumericTokenStream, TestPackedTokenAttributeImpl, TestManyFields, TestNumericDocValuesUpdates, TestGraphTokenizers, Test2BPagedBytes, TestTimSorter, TestEliasFanoDocIdSet, TestInfoStream, TestBlockPostingsFormat3, TestRegexpQuery, TestDocValuesRangeQuery, TestDocTermOrdsRewriteMethod, TestSloppyPhraseQuery2, TestMockDirectoryWrapper, TestBlockPostingsFormat, TestCharsRefBuilder, TestIndexWriterMergePolicy, TestBooleanQuery, TestMinShouldMatch2, TestTerm, TestIntsRef, TestAssertions, TestFieldMaskingSpanQuery, TestPostingsOffsets, TestArrayUtil, TestLongPostings, TestIndexWriterForceMerge]
   [junit4] Completed on J1 in 6.51s, 4 tests, 1 failure <<< FAILURES!

[...truncated 663 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:49: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 421 suites, 3373 tests, 1 failure, 40 ignored (30 assumptions)

Total time: 48 minutes 55 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 3647849 bytes
Compression is 0.0%
Took 1.1 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 773 - Still Failing

Posted by Adrien Grand <jp...@gmail.com>.
It was an integer overflow on the test. I committer a fix.

On Sun, Mar 1, 2015 at 8:22 PM, Apache Jenkins Server
<je...@builds.apache.org> wrote:
> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/773/
>
> 1 tests failed.
> REGRESSION:  org.apache.lucene.util.TestTimSorter.testWorstCaseStackSize
>
> Error Message:
> 31825915
>
> Stack Trace:
> java.lang.ArrayIndexOutOfBoundsException: 31825915
>         at __randomizedtesting.SeedInfo.seed([4053C53E0D183AF1:6166E0C9A7B1CE44]:0)
>         at org.apache.lucene.util.packed.Packed64SingleBlock$Packed64SingleBlock1.set(Packed64SingleBlock.java:272)
>         at org.apache.lucene.util.TestTimSorter.createArray(TestTimSorter.java:88)
>         at org.apache.lucene.util.TestTimSorter.generateWorstCaseArray(TestTimSorter.java:98)
>         at org.apache.lucene.util.TestTimSorter.testWorstCaseStackSize(TestTimSorter.java:46)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
>
>
> Build Log:
> [...truncated 431 lines...]
>    [junit4] Suite: org.apache.lucene.util.TestTimSorter
>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestTimSorter -Dtests.method=testWorstCaseStackSize -Dtests.seed=4053C53E0D183AF1 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=ar_LB -Dtests.timezone=Iran -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
>    [junit4] ERROR   0.28s J1 | TestTimSorter.testWorstCaseStackSize <<<
>    [junit4]    > Throwable #1: java.lang.ArrayIndexOutOfBoundsException: 31825915
>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([4053C53E0D183AF1:6166E0C9A7B1CE44]:0)
>    [junit4]    >        at org.apache.lucene.util.packed.Packed64SingleBlock$Packed64SingleBlock1.set(Packed64SingleBlock.java:272)
>    [junit4]    >        at org.apache.lucene.util.TestTimSorter.createArray(TestTimSorter.java:88)
>    [junit4]    >        at org.apache.lucene.util.TestTimSorter.generateWorstCaseArray(TestTimSorter.java:98)
>    [junit4]    >        at org.apache.lucene.util.TestTimSorter.testWorstCaseStackSize(TestTimSorter.java:46)
>    [junit4]    >        at java.lang.Thread.run(Thread.java:745)
>    [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=DefaultSimilarity, locale=ar_LB, timezone=Iran
>    [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=43990440,total=466616320
>    [junit4]   2> NOTE: All tests run in this JVM: [TestArrayUtil, TestPackedTokenAttributeImpl, TestBlockPostingsFormat3, TestPerSegmentDeletes, TestInfoStream, TestIntsRef, TestLockFactory, TestEliasFanoDocIdSet, TestLiveFieldValues, TestDateFilter, TestBooleanScorer, TestAutomatonQueryUnicode, TestFilteredSearch, TestTopDocsCollector, TestFieldCacheRewriteMethod, TestFuzzyQuery, TestPayloadExplanations, TestCharsRefBuilder, TestStressAdvance, TestAssertions, TestMultiTermConstantScore, TestPriorityQueue, TestAddIndexes, TestManyFields, TestTimSorter]
>    [junit4] Completed on J1 in 0.42s, 10 tests, 1 error <<< FAILURES!
>
> [...truncated 1304 lines...]
> BUILD FAILED
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:49: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 421 suites, 3353 tests, 1 error, 39 ignored (29 assumptions)
>
> Total time: 43 minutes 50 seconds
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
> Archived 5 artifacts
> Archive block size is 32768
> Received 0 blocks and 3621915 bytes
> Compression is 0.0%
> Took 0.68 sec
> Recording test results
> Email was triggered for: Failure
> Sending email for trigger: Failure
>
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org



-- 
Adrien

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 773 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/773/

1 tests failed.
REGRESSION:  org.apache.lucene.util.TestTimSorter.testWorstCaseStackSize

Error Message:
31825915

Stack Trace:
java.lang.ArrayIndexOutOfBoundsException: 31825915
	at __randomizedtesting.SeedInfo.seed([4053C53E0D183AF1:6166E0C9A7B1CE44]:0)
	at org.apache.lucene.util.packed.Packed64SingleBlock$Packed64SingleBlock1.set(Packed64SingleBlock.java:272)
	at org.apache.lucene.util.TestTimSorter.createArray(TestTimSorter.java:88)
	at org.apache.lucene.util.TestTimSorter.generateWorstCaseArray(TestTimSorter.java:98)
	at org.apache.lucene.util.TestTimSorter.testWorstCaseStackSize(TestTimSorter.java:46)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)




Build Log:
[...truncated 431 lines...]
   [junit4] Suite: org.apache.lucene.util.TestTimSorter
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestTimSorter -Dtests.method=testWorstCaseStackSize -Dtests.seed=4053C53E0D183AF1 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=ar_LB -Dtests.timezone=Iran -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.28s J1 | TestTimSorter.testWorstCaseStackSize <<<
   [junit4]    > Throwable #1: java.lang.ArrayIndexOutOfBoundsException: 31825915
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([4053C53E0D183AF1:6166E0C9A7B1CE44]:0)
   [junit4]    > 	at org.apache.lucene.util.packed.Packed64SingleBlock$Packed64SingleBlock1.set(Packed64SingleBlock.java:272)
   [junit4]    > 	at org.apache.lucene.util.TestTimSorter.createArray(TestTimSorter.java:88)
   [junit4]    > 	at org.apache.lucene.util.TestTimSorter.generateWorstCaseArray(TestTimSorter.java:98)
   [junit4]    > 	at org.apache.lucene.util.TestTimSorter.testWorstCaseStackSize(TestTimSorter.java:46)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {}, docValues:{}, sim=DefaultSimilarity, locale=ar_LB, timezone=Iran
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=43990440,total=466616320
   [junit4]   2> NOTE: All tests run in this JVM: [TestArrayUtil, TestPackedTokenAttributeImpl, TestBlockPostingsFormat3, TestPerSegmentDeletes, TestInfoStream, TestIntsRef, TestLockFactory, TestEliasFanoDocIdSet, TestLiveFieldValues, TestDateFilter, TestBooleanScorer, TestAutomatonQueryUnicode, TestFilteredSearch, TestTopDocsCollector, TestFieldCacheRewriteMethod, TestFuzzyQuery, TestPayloadExplanations, TestCharsRefBuilder, TestStressAdvance, TestAssertions, TestMultiTermConstantScore, TestPriorityQueue, TestAddIndexes, TestManyFields, TestTimSorter]
   [junit4] Completed on J1 in 0.42s, 10 tests, 1 error <<< FAILURES!

[...truncated 1304 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:49: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 421 suites, 3353 tests, 1 error, 39 ignored (29 assumptions)

Total time: 43 minutes 50 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 3621915 bytes
Compression is 0.0%
Took 0.68 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 772 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/772/

6 tests failed.
FAILED:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:36481/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:36481/collection1
	at __randomizedtesting.SeedInfo.seed([83FA1E8E8B0C749E:BAE215425F01966]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:131)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:309)
	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:224)
	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:166)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:671)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test(FullSolrCloudDistribCmdsTest.java:151)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:945)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:920)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketException: Broken pipe
	at java.net.SocketOutputStream.socketWrite0(Native Method)
	at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:113)
	at java.net.SocketOutputStream.write(SocketOutputStream.java:159)
	at org.apache.http.impl.io.AbstractSessionOutputBuffer.flushBuffer(AbstractSessionOutputBuffer.java:159)
	at org.apache.http.impl.io.AbstractSessionOutputBuffer.writeLine(AbstractSessionOutputBuffer.java:258)
	at org.apache.http.impl.io.HttpRequestWriter.writeHeadLine(HttpRequestWriter.java:59)
	at org.apache.http.impl.io.HttpRequestWriter.writeHeadLine(HttpRequestWriter.java:46)
	at org.apache.http.impl.io.AbstractMessageWriter.write(AbstractMessageWriter.java:109)
	at org.apache.http.impl.AbstractHttpClientConnection.sendRequestHeader(AbstractHttpClientConnection.java:254)
	at org.apache.http.impl.conn.DefaultClientConnection.sendRequestHeader(DefaultClientConnection.java:270)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.sendRequestHeader(ManagedClientConnectionImpl.java:209)
	at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:203)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:121)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 50 more


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([83FA1E8E8B0C749E:BAE215425F01966]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:899)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:752)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:945)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:920)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:568)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:565)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([83FA1E8E8B0C749E:BAE215425F01966]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:899)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:752)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:945)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:920)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:568)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:565)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:15988/c8n_1x3_commits_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:15988/c8n_1x3_commits_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([83FA1E8E8B0C749E:BAE215425F01966]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:131)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:176)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:154)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:945)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:920)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 49 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([83FA1E8E8B0C749E:BAE215425F01966]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:899)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:752)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:131)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:110)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:945)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:920)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:568)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:565)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.http.NoHttpResponseException: The target server failed to respond

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
	at __randomizedtesting.SeedInfo.seed([83FA1E8E8B0C749E:BAE215425F01966]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:899)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:752)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:945)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:920)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:568)
	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:565)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	... 1 more




Build Log:
[...truncated 10484 lines...]
   [junit4] Suite: org.apache.solr.cloud.RecoveryAfterSoftCommitTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/init-core-data-001
   [junit4]   2> 871870 T2363 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /ss_to/t
   [junit4]   2> 871877 T2363 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 871877 T2364 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 871977 T2363 oasc.ZkTestServer.run start zk server on port:20283
   [junit4]   2> 871978 T2363 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 871979 T2363 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 871983 T2371 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@10cab054 name:ZooKeeperConnection Watcher:127.0.0.1:20283 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 871983 T2363 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 871983 T2363 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 871983 T2363 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 871986 T2363 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 871987 T2363 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 871988 T2374 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@3e81250f name:ZooKeeperConnection Watcher:127.0.0.1:20283/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 871989 T2363 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 871989 T2363 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 871990 T2363 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 871992 T2363 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 871994 T2363 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 871995 T2363 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 871998 T2363 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 871999 T2363 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 872002 T2363 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 872003 T2363 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 872005 T2363 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 872006 T2363 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 872008 T2363 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 872008 T2363 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 872010 T2363 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 872011 T2363 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 872013 T2363 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 872014 T2363 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 872016 T2363 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 872016 T2363 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 872019 T2363 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 872019 T2363 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 872022 T2363 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 872022 T2363 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 872025 T2363 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 872025 T2363 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 872027 T2363 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 872028 T2363 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 873834 T2363 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/control-001/cores/collection1
   [junit4]   2> 873840 T2363 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 873844 T2363 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:20290
   [junit4]   2> 873844 T2363 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 873845 T2363 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 873845 T2363 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/control-001
   [junit4]   2> 873846 T2363 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/control-001/'
   [junit4]   2> 873901 T2363 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/control-001/solr.xml
   [junit4]   2> 873975 T2363 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/control-001/cores
   [junit4]   2> 873976 T2363 oasc.CoreContainer.<init> New CoreContainer 1805030825
   [junit4]   2> 873976 T2363 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/control-001/]
   [junit4]   2> 873977 T2363 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/control-001/lib
   [junit4]   2> 873978 T2363 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/control-001/lib).
   [junit4]   2> 873990 T2363 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 873991 T2363 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 873991 T2363 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 873991 T2363 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 873992 T2363 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 873992 T2363 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 873992 T2363 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 873993 T2363 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 873993 T2363 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 873993 T2363 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 873994 T2363 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 873994 T2363 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 873995 T2363 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 873995 T2363 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 873995 T2363 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 873996 T2363 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:20283/solr
   [junit4]   2> 873996 T2363 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 873996 T2363 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 873997 T2363 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 873999 T2386 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@21e0b70c name:ZooKeeperConnection Watcher:127.0.0.1:20283 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 874000 T2363 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 874001 T2363 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 874003 T2363 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 874005 T2389 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@7e78f76c name:ZooKeeperConnection Watcher:127.0.0.1:20283/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 874005 T2363 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 874007 T2363 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 874010 T2363 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 874013 T2363 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 874016 T2363 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 874019 T2363 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 874022 T2363 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 874023 T2363 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:20289_ss_to%2Ft
   [junit4]   2> 874024 T2363 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:20289_ss_to%2Ft
   [junit4]   2> 874027 T2363 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 874029 T2363 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 874031 T2363 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 874033 T2363 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:20289_ss_to%2Ft
   [junit4]   2> 874034 T2363 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 874036 T2363 oasc.Overseer.start Overseer (id=93397866809131011-127.0.0.1:20289_ss_to%2Ft-n_0000000000) starting
   [junit4]   2> 874038 T2363 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 874047 T2363 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 874048 T2391 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 874049 T2363 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 874049 T2390 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 874051 T2363 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 874053 T2363 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 874058 T2363 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/control-001/cores
   [junit4]   2> 874061 T2363 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/control-001/cores/collection1/
   [junit4]   2> 874061 T2363 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 874062 T2393 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 874062 T2393 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 874064 T2389 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 874064 T2393 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 874065 T2390 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20289/ss_to/t",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:20289_ss_to%2Ft",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down"} current state version: 0
   [junit4]   2> 874066 T2390 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20289/ss_to/t",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:20289_ss_to%2Ft",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 874066 T2390 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 874066 T2390 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 874068 T2389 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 875064 T2393 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 875065 T2393 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 875066 T2393 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 875066 T2393 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 875067 T2393 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 875067 T2393 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/control-001/cores/collection1/'
   [junit4]   2> 875084 T2393 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 875088 T2393 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 875101 T2393 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 875128 T2393 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 875184 T2393 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 875185 T2393 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 875198 T2393 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 875641 T2393 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 875655 T2393 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 875658 T2393 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 875675 T2393 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 875680 T2393 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 875685 T2393 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 875686 T2393 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 875686 T2393 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 875687 T2393 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 875688 T2393 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 875688 T2393 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 875688 T2393 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 875688 T2393 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 875689 T2393 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/tempDir-001/control/data/
   [junit4]   2> 875689 T2393 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@c61a8f8
   [junit4]   2> 875690 T2393 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/tempDir-001/control/data
   [junit4]   2> 875690 T2393 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/tempDir-001/control/data/index/
   [junit4]   2> 875690 T2393 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 875691 T2393 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/tempDir-001/control/data/index
   [junit4]   2> 875691 T2393 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.MockRandomMergePolicy: org.apache.lucene.index.MockRandomMergePolicy@2fa8302e
   [junit4]   2> 875692 T2393 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4af33473 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@24f89076),segFN=segments_1,generation=1}
   [junit4]   2> 875692 T2393 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 875697 T2393 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 875698 T2393 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 875698 T2393 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 875698 T2393 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 875698 T2393 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 875699 T2393 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 875699 T2393 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 875699 T2393 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 875700 T2393 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 875700 T2393 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 875701 T2393 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 875701 T2393 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 875702 T2393 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 875702 T2393 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 875703 T2393 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 875703 T2393 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 875704 T2393 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 875704 T2393 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 875706 T2393 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 875706 T2393 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 875707 T2393 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 875707 T2393 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 875708 T2393 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 875708 T2393 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 875709 T2393 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 875709 T2393 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 875710 T2393 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 875738 T2393 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 875741 T2393 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 875743 T2393 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 875745 T2393 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 875758 T2393 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 875765 T2393 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 875768 T2393 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 875768 T2393 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 875769 T2393 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=36, maxMergeAtOnceExplicit=37, maxMergedSegmentMB=51.0341796875, floorSegmentMB=1.169921875, forceMergeDeletesPctAllowed=21.111540099733464, segmentsPerTier=17.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=1.0
   [junit4]   2> 875769 T2393 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=MockDirectoryWrapper(RAMDirectory@4af33473 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@24f89076),segFN=segments_1,generation=1}
   [junit4]   2> 875770 T2393 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 875770 T2393 oass.SolrIndexSearcher.<init> Opening Searcher@6d73cdff[collection1] main
   [junit4]   2> 875770 T2393 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 875772 T2393 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 875772 T2393 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 875773 T2393 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 875773 T2393 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 875773 T2393 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 875774 T2393 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 875774 T2393 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 875775 T2393 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 875775 T2393 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 875776 T2394 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@6d73cdff[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 875776 T2393 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 875777 T2397 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:20289/ss_to/t collection:control_collection shard:shard1
   [junit4]   2> 875778 T2363 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2
   [junit4]   2> 875778 T2363 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 875779 T2397 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 875782 T2363 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 875783 T2363 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 875785 T2401 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@249030f9 name:ZooKeeperConnection Watcher:127.0.0.1:20283/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 875785 T2363 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 875786 T2397 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 875786 T2363 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 875788 T2363 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 875789 T2389 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 875789 T2397 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 875789 T2397 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C272 name=collection1 org.apache.solr.core.SolrCore@7bcd69f3 url=http://127.0.0.1:20289/ss_to/t/collection1 node=127.0.0.1:20289_ss_to%2Ft C272_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:20289/ss_to/t, node_name=127.0.0.1:20289_ss_to%2Ft, core=collection1, state=down}
   [junit4]   2> 875789 T2397 C272 P20289 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:20289/ss_to/t/collection1/
   [junit4]   2> 875790 T2390 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 875790 T2397 C272 P20289 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 875790 T2397 C272 P20289 oasc.SyncStrategy.syncToMe http://127.0.0.1:20289/ss_to/t/collection1/ has no replicas
   [junit4]   2> 875790 T2397 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:20289/ss_to/t/collection1/ shard1
   [junit4]   2> 875791 T2397 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 875792 T2389 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 875792 T2401 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 875793 T2363 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 875797 T2389 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 875799 T2390 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20289/ss_to/t",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 875904 T2389 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 875904 T2401 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 875952 T2397 oasc.ZkController.register We are http://127.0.0.1:20289/ss_to/t/collection1/ and leader is http://127.0.0.1:20289/ss_to/t/collection1/
   [junit4]   2> 875953 T2397 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:20289/ss_to/t
   [junit4]   2> 875953 T2397 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 875953 T2397 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 875953 T2397 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 875955 T2389 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 875956 T2390 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20289/ss_to/t",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:20289_ss_to%2Ft",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 3
   [junit4]   2> 875957 T2390 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20289/ss_to/t",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:20289_ss_to%2Ft",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"}
   [junit4]   2> 876061 T2401 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 876061 T2389 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 877375 T2363 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/shard-1-001/cores/collection1
   [junit4]   2> 877377 T2363 oasc.AbstractFullDistribZkTestBase.createJettys create jetty 1
   [junit4]   2> 877378 T2363 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 877380 T2363 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:20312
   [junit4]   2> 877380 T2363 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@57bd06bf
   [junit4]   2> 877381 T2363 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 877381 T2363 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/shard-1-001
   [junit4]   2> 877381 T2363 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/shard-1-001/'
   [junit4]   2> 877410 T2363 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/shard-1-001/solr.xml
   [junit4]   2> 877445 T2363 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/shard-1-001/cores
   [junit4]   2> 877445 T2363 oasc.CoreContainer.<init> New CoreContainer 22886361
   [junit4]   2> 877445 T2363 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/shard-1-001/]
   [junit4]   2> 877446 T2363 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/shard-1-001/lib
   [junit4]   2> 877446 T2363 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/shard-1-001/lib).
   [junit4]   2> 877455 T2363 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 877455 T2363 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 877455 T2363 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 877455 T2363 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 877456 T2363 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 877456 T2363 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 877456 T2363 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 877456 T2363 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 877456 T2363 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 877457 T2363 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 877457 T2363 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 877457 T2363 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 877458 T2363 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 877458 T2363 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 877458 T2363 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 877458 T2363 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:20283/solr
   [junit4]   2> 877459 T2363 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 877459 T2363 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 877459 T2363 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 877461 T2413 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2daa8854 name:ZooKeeperConnection Watcher:127.0.0.1:20283 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 877462 T2363 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 877462 T2363 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 877464 T2363 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 877466 T2416 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@5e7a1b6d name:ZooKeeperConnection Watcher:127.0.0.1:20283/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 877466 T2363 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 877472 T2363 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 878476 T2363 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:20311_ss_to%2Ft
   [junit4]   2> 878477 T2363 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:20311_ss_to%2Ft
   [junit4]   2> 878481 T2363 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 878483 T2363 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/shard-1-001/cores
   [junit4]   2> 878485 T2363 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.RecoveryAfterSoftCommitTest 83FA1E8E8B0C749E-001/shard-1-001/cores/collection1/
   [junit4]   2> 878485 T2363 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 878486 T2417 oasc.ZkController.publish publishing core=collection1 state=down collection=collection1
   [junit4]   2> 878486 T2417 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 878487 T2389 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 878487 T2417 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 878488 T2390 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20311/ss_to/t",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:20311_ss_to%2Ft",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down"} current state version: 4
   [junit4]   2> 878489 T2390 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "collection":"collection1",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "base_url":"http://127.0.0.1:20311/ss_to/t",
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "node_name":"127.0.0.1:20311_ss_to%2Ft",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"down"}
   [junit4]   2> 878489 T2390 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 878490 T2390 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 878491 T2389 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 878491 T2401 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 878491 T2416 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 2)
   [junit4]   2> 879488 T2417 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate 

[...truncated too long message...]

xy.close WARN Closing 0 connections to: http://127.0.0.1:35978/
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=83FA1E8E8B0C749E -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=sr_ME_#Latn -Dtests.timezone=Antarctica/South_Pole -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   71.7s J2 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([83FA1E8E8B0C749E:BAE215425F01966]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:899)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:752)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:945)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:920)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:568)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient$2.call(CloudSolrClient.java:565)
   [junit4]    > 	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
   [junit4]    > 	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
   [junit4]    > 	... 1 more
   [junit4]   2> 2434167 T11138 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest 83FA1E8E8B0C749E-001
   [junit4]   2> 71723 T11137 ccr.ThreadLeakControl.checkThreadLeaks WARNING Will linger awaiting termination of 1 leaked thread(s).
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {timestamp=PostingsFormat(name=LuceneFixedGap), multiDefault=PostingsFormat(name=Memory doPackFST= true), _version_=PostingsFormat(name=LuceneFixedGap), intDefault=PostingsFormat(name=LuceneFixedGap), id=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128))), range_facet_l=PostingsFormat(name=LuceneFixedGap), text=FST50, a_t=PostingsFormat(name=LuceneFixedGap)}, docValues:{timestamp=DocValuesFormat(name=Lucene50)}, sim=DefaultSimilarity, locale=sr_ME_#Latn, timezone=Antarctica/South_Pole
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=127441176,total=485490688
   [junit4]   2> NOTE: All tests run in this JVM: [TestLRUCache, DeleteShardTest, TestAnalyzeInfixSuggestions, TestRealTimeGet, TestHashPartitioner, ResponseHeaderTest, TestPerFieldSimilarity, RAMDirectoryFactoryTest, TestRTGBase, TestInfoStreamLogging, SolrRequestParserTest, TestTolerantSearch, TestBinaryField, SuggestComponentTest, TestDistributedMissingSort, TestLRUStatsCache, TestLazyCores, TestFastWriter, TestDistributedSearch, DirectUpdateHandlerOptimizeTest, SimpleMLTQParserTest, NoCacheHeaderTest, SyncSliceTest, FileBasedSpellCheckerTest, TestInitQParser, RecoveryAfterSoftCommitTest, SolrPluginUtilsTest, TestSuggestSpellingConverter, TestAnalyzedSuggestions, SpellCheckComponentTest, SolrXmlInZkTest, HardAutoCommitTest, TestCSVResponseWriter, SSLMigrationTest, DateFieldTest, DistributedSuggestComponentTest, TestReload, TestFieldSortValues, TestPHPSerializedResponseWriter, DistributedFacetPivotWhiteBoxTest, CurrencyFieldOpenExchangeTest, URLClassifyProcessorTest, HdfsChaosMonkeySafeLeaderTest, TestExtendedDismaxParser, NotRequiredUniqueKeyTest, TestRemoteStreaming, TestReplicationHandlerBackup, BlockDirectoryTest, TestSolrQueryParser, TestClusterStateMutator, TestSerializedLuceneMatchVersion, TestMergePolicyConfig, IgnoreCommitOptimizeUpdateProcessorFactoryTest, ReturnFieldsTest, BasicDistributedZk2Test, LeaderElectionIntegrationTest, FullSolrCloudDistribCmdsTest, ClusterStateUpdateTest, ZkSolrClientTest, TestRandomDVFaceting, ZkCLITest, AutoCommitTest, TestFaceting, DistributedSpellCheckComponentTest, ZkControllerTest, TestJoin, TestGroupingSearch, SimplePostToolTest, AnalysisAfterCoreReloadTest, DocValuesTest, SuggesterTSTTest, PolyFieldTest, FieldMutatingUpdateProcessorTest, QueryEqualityTest, CacheHeaderTest, TestQueryUtils, TestReversedWildcardFilterFactory, RequestHandlersTest, TermVectorComponentTest, RequiredFieldsTest, IndexSchemaRuntimeFieldTest, TestJmxIntegration, QueryParsingTest, PingRequestHandlerTest, HighlighterConfigTest, AlternateDirectoryTest, TestSolrIndexConfig, SolrIndexConfigTest, TestElisionMultitermQuery, TestLuceneMatchVersion, TestXIncludeConfig, TestBM25SimilarityFactory, DOMUtilTest, ClusterStateTest, TestDocumentBuilder, SystemInfoHandlerTest, DistributedMLTComponentTest, DistributedIntervalFacetingTest, CollectionsAPIAsyncDistributedZkTest, ConnectionManagerTest, CustomCollectionTest, DistribCursorPagingTest, DistribJoinFromCollectionTest, LeaderInitiatedRecoveryOnCommitTest, MigrateRouteKeyTest, OverriddenZkACLAndCredentialsProvidersTest, OverseerStatusTest, ReplicationFactorTest]
   [junit4] Completed on J2 in 72.02s, 1 test, 1 error <<< FAILURES!

[...truncated 294 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 468 suites, 1870 tests, 6 errors, 58 ignored (9 assumptions)

Total time: 161 minutes 5 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 770166182 bytes
Compression is 0.0%
Took 2 min 24 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 771 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/771/

7 tests failed.
FAILED:  org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:60184/lvl/jl/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:60184/lvl/jl/collection1
	at __randomizedtesting.SeedInfo.seed([1F64C8A296BD6261:9730F77838410F99]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:131)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:309)
	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:224)
	at org.apache.solr.cloud.CloudInspectUtil.compareResults(CloudInspectUtil.java:166)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.testIndexingBatchPerRequestWithHttpSolrClient(FullSolrCloudDistribCmdsTest.java:671)
	at org.apache.solr.cloud.FullSolrCloudDistribCmdsTest.test(FullSolrCloudDistribCmdsTest.java:151)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:945)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:920)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketException: Broken pipe
	at java.net.SocketOutputStream.socketWrite0(Native Method)
	at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:113)
	at java.net.SocketOutputStream.write(SocketOutputStream.java:159)
	at org.apache.http.impl.io.AbstractSessionOutputBuffer.flushBuffer(AbstractSessionOutputBuffer.java:159)
	at org.apache.http.impl.io.AbstractSessionOutputBuffer.writeLine(AbstractSessionOutputBuffer.java:258)
	at org.apache.http.impl.io.HttpRequestWriter.writeHeadLine(HttpRequestWriter.java:59)
	at org.apache.http.impl.io.HttpRequestWriter.writeHeadLine(HttpRequestWriter.java:46)
	at org.apache.http.impl.io.AbstractMessageWriter.write(AbstractMessageWriter.java:109)
	at org.apache.http.impl.AbstractHttpClientConnection.sendRequestHeader(AbstractHttpClientConnection.java:254)
	at org.apache.http.impl.conn.DefaultClientConnection.sendRequestHeader(DefaultClientConnection.java:270)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.sendRequestHeader(ManagedClientConnectionImpl.java:209)
	at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:203)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:121)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 50 more


FAILED:  org.apache.solr.cloud.HttpPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:52988/c8n_1x2_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:52988/c8n_1x2_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([1F64C8A296BD6261:9730F77838410F99]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:597)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:918)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:809)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:752)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.HttpPartitionTest.testRf2(HttpPartitionTest.java:193)
	at org.apache.solr.cloud.HttpPartitionTest.test(HttpPartitionTest.java:106)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:945)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:920)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:52988/c8n_1x2_shard1_replica1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:594)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37043/g_muw/p/c8n_1x3_lf_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37043/g_muw/p/c8n_1x3_lf_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([1F64C8A296BD6261:9730F77838410F99]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:597)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:918)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:809)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:752)
	at org.apache.solr.cloud.HttpPartitionTest.doSendDoc(HttpPartitionTest.java:484)
	at org.apache.solr.cloud.HttpPartitionTest.sendDoc(HttpPartitionTest.java:501)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.testRf3WithLeaderFailover(LeaderFailoverAfterPartitionTest.java:78)
	at org.apache.solr.cloud.LeaderFailoverAfterPartitionTest.test(LeaderFailoverAfterPartitionTest.java:51)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:945)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:920)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37043/g_muw/p/c8n_1x3_lf_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:594)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 54 more


FAILED:  org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test

Error Message:
IOException occured when talking to server at: http://127.0.0.1:37096/c8n_1x3_commits_shard1_replica1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37096/c8n_1x3_commits_shard1_replica1
	at __randomizedtesting.SeedInfo.seed([1F64C8A296BD6261:9730F77838410F99]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:131)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:176)
	at org.apache.solr.client.solrj.SolrClient.commit(SolrClient.java:154)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.oneShardTest(LeaderInitiatedRecoveryOnCommitTest.java:130)
	at org.apache.solr.cloud.LeaderInitiatedRecoveryOnCommitTest.test(LeaderInitiatedRecoveryOnCommitTest.java:62)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:945)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:920)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 49 more


FAILED:  org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37384/collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37384/collection1
	at __randomizedtesting.SeedInfo.seed([1F64C8A296BD6261:9730F77838410F99]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:597)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:918)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:809)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:752)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:131)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:124)
	at org.apache.solr.client.solrj.SolrClient.add(SolrClient.java:110)
	at org.apache.solr.cloud.RecoveryAfterSoftCommitTest.test(RecoveryAfterSoftCommitTest.java:88)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:945)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:920)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37384/collection1
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:594)
	... 49 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 54 more


FAILED:  org.apache.solr.cloud.ReplicationFactorTest.test

Error Message:
org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37433/kcd/zh/repfacttest_c8n_1x3_shard1_replica2

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37433/kcd/zh/repfacttest_c8n_1x3_shard1_replica2
	at __randomizedtesting.SeedInfo.seed([1F64C8A296BD6261:9730F77838410F99]:0)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:597)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:918)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:809)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:752)
	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:945)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:920)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37433/kcd/zh/repfacttest_c8n_1x3_shard1_replica2
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:594)
	... 48 more
Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 53 more


FAILED:  org.apache.solr.handler.component.DistributedMLTComponentTest.test

Error Message:
Timeout occured while waiting response from server at: http://127.0.0.1:18915//collection1

Stack Trace:
org.apache.solr.client.solrj.SolrServerException: Timeout occured while waiting response from server at: http://127.0.0.1:18915//collection1
	at __randomizedtesting.SeedInfo.seed([1F64C8A296BD6261:9730F77838410F99]:0)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:568)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
	at org.apache.solr.client.solrj.SolrRequest.process(SolrRequest.java:131)
	at org.apache.solr.client.solrj.SolrClient.query(SolrClient.java:309)
	at org.apache.solr.BaseDistributedSearchTestCase.queryServer(BaseDistributedSearchTestCase.java:543)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:591)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:573)
	at org.apache.solr.BaseDistributedSearchTestCase.query(BaseDistributedSearchTestCase.java:552)
	at org.apache.solr.handler.component.DistributedMLTComponentTest.test(DistributedMLTComponentTest.java:126)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:945)
	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:920)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule$1.evaluate(SystemPropertiesRestoreRule.java:53)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
	at java.net.SocketInputStream.socketRead0(Native Method)
	at java.net.SocketInputStream.read(SocketInputStream.java:152)
	at java.net.SocketInputStream.read(SocketInputStream.java:122)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.fillBuffer(AbstractSessionInputBuffer.java:160)
	at org.apache.http.impl.io.SocketInputBuffer.fillBuffer(SocketInputBuffer.java:84)
	at org.apache.http.impl.io.AbstractSessionInputBuffer.readLine(AbstractSessionInputBuffer.java:273)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
	... 51 more




Build Log:
[...truncated 10245 lines...]
   [junit4] Suite: org.apache.solr.cloud.HttpPartitionTest
   [junit4]   2> Creating dataDir: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 1F64C8A296BD6261-001/init-core-data-001
   [junit4]   2> 319882 T1394 oas.BaseDistributedSearchTestCase.initHostContext Setting hostContext system property: /
   [junit4]   2> 319890 T1394 oasc.ZkTestServer.run STARTING ZK TEST SERVER
   [junit4]   1> client port:0.0.0.0/0.0.0.0:0
   [junit4]   2> 319891 T1395 oasc.ZkTestServer$ZKServerMain.runFromConfig Starting server
   [junit4]   2> 319991 T1394 oasc.ZkTestServer.run start zk server on port:52978
   [junit4]   2> 319992 T1394 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 319993 T1394 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 319999 T1402 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@769d4645 name:ZooKeeperConnection Watcher:127.0.0.1:52978 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 320000 T1394 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 320001 T1394 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 320001 T1394 oascc.SolrZkClient.makePath makePath: /solr
   [junit4]   2> 320006 T1394 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 320008 T1394 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 320010 T1405 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@6ed86a75 name:ZooKeeperConnection Watcher:127.0.0.1:52978/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 320010 T1394 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 320011 T1394 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 320011 T1394 oascc.SolrZkClient.makePath makePath: /collections/collection1
   [junit4]   2> 320014 T1394 oascc.SolrZkClient.makePath makePath: /collections/collection1/shards
   [junit4]   2> 320017 T1394 oascc.SolrZkClient.makePath makePath: /collections/control_collection
   [junit4]   2> 320020 T1394 oascc.SolrZkClient.makePath makePath: /collections/control_collection/shards
   [junit4]   2> 320022 T1394 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml to /configs/conf1/solrconfig.xml
   [junit4]   2> 320023 T1394 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.xml
   [junit4]   2> 320027 T1394 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/schema.xml to /configs/conf1/schema.xml
   [junit4]   2> 320028 T1394 oascc.SolrZkClient.makePath makePath: /configs/conf1/schema.xml
   [junit4]   2> 320032 T1394 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/solrconfig.snippet.randomindexconfig.xml to /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 320032 T1394 oascc.SolrZkClient.makePath makePath: /configs/conf1/solrconfig.snippet.randomindexconfig.xml
   [junit4]   2> 320035 T1394 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/stopwords.txt to /configs/conf1/stopwords.txt
   [junit4]   2> 320036 T1394 oascc.SolrZkClient.makePath makePath: /configs/conf1/stopwords.txt
   [junit4]   2> 320039 T1394 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/protwords.txt to /configs/conf1/protwords.txt
   [junit4]   2> 320040 T1394 oascc.SolrZkClient.makePath makePath: /configs/conf1/protwords.txt
   [junit4]   2> 320043 T1394 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/currency.xml to /configs/conf1/currency.xml
   [junit4]   2> 320044 T1394 oascc.SolrZkClient.makePath makePath: /configs/conf1/currency.xml
   [junit4]   2> 320047 T1394 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/enumsConfig.xml to /configs/conf1/enumsConfig.xml
   [junit4]   2> 320048 T1394 oascc.SolrZkClient.makePath makePath: /configs/conf1/enumsConfig.xml
   [junit4]   2> 320050 T1394 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/open-exchange-rates.json to /configs/conf1/open-exchange-rates.json
   [junit4]   2> 320052 T1394 oascc.SolrZkClient.makePath makePath: /configs/conf1/open-exchange-rates.json
   [junit4]   2> 320054 T1394 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/mapping-ISOLatin1Accent.txt to /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 320055 T1394 oascc.SolrZkClient.makePath makePath: /configs/conf1/mapping-ISOLatin1Accent.txt
   [junit4]   2> 320057 T1394 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/old_synonyms.txt to /configs/conf1/old_synonyms.txt
   [junit4]   2> 320058 T1394 oascc.SolrZkClient.makePath makePath: /configs/conf1/old_synonyms.txt
   [junit4]   2> 320060 T1394 oasc.AbstractZkTestCase.putConfig put /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/core/src/test-files/solr/collection1/conf/synonyms.txt to /configs/conf1/synonyms.txt
   [junit4]   2> 320061 T1394 oascc.SolrZkClient.makePath makePath: /configs/conf1/synonyms.txt
   [junit4]   2> 322055 T1394 oas.SolrTestCaseJ4.writeCoreProperties Writing core.properties file to /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 1F64C8A296BD6261-001/control-001/cores/collection1
   [junit4]   2> 322060 T1394 oejs.Server.doStart jetty-8.1.10.v20130312
   [junit4]   2> 322065 T1394 oejs.AbstractConnector.doStart Started SocketConnector@127.0.0.1:52982
   [junit4]   2> 322066 T1394 oass.SolrDispatchFilter.init SolrDispatchFilter.init()sun.misc.Launcher$AppClassLoader@3c56b64c
   [junit4]   2> 322066 T1394 oasc.SolrResourceLoader.locateSolrHome JNDI not configured for solr (NoInitialContextEx)
   [junit4]   2> 322067 T1394 oasc.SolrResourceLoader.locateSolrHome using system property solr.solr.home: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 1F64C8A296BD6261-001/control-001
   [junit4]   2> 322067 T1394 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 1F64C8A296BD6261-001/control-001/'
   [junit4]   2> 322125 T1394 oasc.SolrXmlConfig.fromFile Loading container configuration from /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 1F64C8A296BD6261-001/control-001/solr.xml
   [junit4]   2> 322202 T1394 oasc.CorePropertiesLocator.<init> Config-defined core root directory: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 1F64C8A296BD6261-001/control-001/cores
   [junit4]   2> 322203 T1394 oasc.CoreContainer.<init> New CoreContainer 1678279759
   [junit4]   2> 322203 T1394 oasc.CoreContainer.load Loading cores into CoreContainer [instanceDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 1F64C8A296BD6261-001/control-001/]
   [junit4]   2> 322203 T1394 oasc.CoreContainer.load loading shared library: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 1F64C8A296BD6261-001/control-001/lib
   [junit4]   2> 322205 T1394 oasc.SolrResourceLoader.addToClassLoader WARN Can't find (or read) directory to add to classloader: lib (resolved as: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 1F64C8A296BD6261-001/control-001/lib).
   [junit4]   2> 322223 T1394 oashc.HttpShardHandlerFactory.getParameter Setting socketTimeout to: 90000
   [junit4]   2> 322223 T1394 oashc.HttpShardHandlerFactory.getParameter Setting urlScheme to: 
   [junit4]   2> 322224 T1394 oashc.HttpShardHandlerFactory.getParameter Setting connTimeout to: 15000
   [junit4]   2> 322224 T1394 oashc.HttpShardHandlerFactory.getParameter Setting maxConnectionsPerHost to: 20
   [junit4]   2> 322224 T1394 oashc.HttpShardHandlerFactory.getParameter Setting maxConnections to: 10000
   [junit4]   2> 322225 T1394 oashc.HttpShardHandlerFactory.getParameter Setting corePoolSize to: 0
   [junit4]   2> 322225 T1394 oashc.HttpShardHandlerFactory.getParameter Setting maximumPoolSize to: 2147483647
   [junit4]   2> 322225 T1394 oashc.HttpShardHandlerFactory.getParameter Setting maxThreadIdleTime to: 5
   [junit4]   2> 322226 T1394 oashc.HttpShardHandlerFactory.getParameter Setting sizeOfQueue to: -1
   [junit4]   2> 322226 T1394 oashc.HttpShardHandlerFactory.getParameter Setting fairnessPolicy to: false
   [junit4]   2> 322227 T1394 oashc.HttpShardHandlerFactory.getParameter Setting useRetries to: false
   [junit4]   2> 322227 T1394 oasu.UpdateShardHandler.<init> Creating UpdateShardHandler HTTP client with params: socketTimeout=340000&connTimeout=45000&retry=true
   [junit4]   2> 322228 T1394 oasl.LogWatcher.createWatcher SLF4J impl is org.slf4j.impl.Log4jLoggerFactory
   [junit4]   2> 322228 T1394 oasl.LogWatcher.newRegisteredLogWatcher Registering Log Listener [Log4j (org.slf4j.impl.Log4jLoggerFactory)]
   [junit4]   2> 322229 T1394 oasc.CoreContainer.load Node Name: 127.0.0.1
   [junit4]   2> 322229 T1394 oasc.ZkContainer.initZooKeeper Zookeeper client=127.0.0.1:52978/solr
   [junit4]   2> 322229 T1394 oasc.ZkController.checkChrootPath zkHost includes chroot
   [junit4]   2> 322230 T1394 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 322231 T1394 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 322234 T1417 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@f134d9d name:ZooKeeperConnection Watcher:127.0.0.1:52978 got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 322234 T1394 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 322236 T1394 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 322239 T1394 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 322242 T1420 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@286a195b name:ZooKeeperConnection Watcher:127.0.0.1:52978/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 322242 T1394 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 322246 T1394 oascc.SolrZkClient.makePath makePath: /overseer/queue
   [junit4]   2> 322250 T1394 oascc.SolrZkClient.makePath makePath: /overseer/collection-queue-work
   [junit4]   2> 322256 T1394 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-running
   [junit4]   2> 322260 T1394 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-completed
   [junit4]   2> 322264 T1394 oascc.SolrZkClient.makePath makePath: /overseer/collection-map-failure
   [junit4]   2> 322268 T1394 oascc.SolrZkClient.makePath makePath: /live_nodes
   [junit4]   2> 322270 T1394 oasc.ZkController.createEphemeralLiveNode Register node as live in ZooKeeper:/live_nodes/127.0.0.1:52981_
   [junit4]   2> 322271 T1394 oascc.SolrZkClient.makePath makePath: /live_nodes/127.0.0.1:52981_
   [junit4]   2> 322275 T1394 oascc.SolrZkClient.makePath makePath: /overseer_elect
   [junit4]   2> 322277 T1394 oascc.SolrZkClient.makePath makePath: /overseer_elect/election
   [junit4]   2> 322279 T1394 oasc.Overseer.close Overseer (id=null) closing
   [junit4]   2> 322282 T1394 oasc.OverseerElectionContext.runLeaderProcess I am going to be the leader 127.0.0.1:52981_
   [junit4]   2> 322283 T1394 oascc.SolrZkClient.makePath makePath: /overseer_elect/leader
   [junit4]   2> 322285 T1394 oasc.Overseer.start Overseer (id=93392119476060163-127.0.0.1:52981_-n_0000000000) starting
   [junit4]   2> 322288 T1394 oascc.SolrZkClient.makePath makePath: /overseer/queue-work
   [junit4]   2> 322299 T1394 oasc.OverseerAutoReplicaFailoverThread.<init> Starting OverseerAutoReplicaFailoverThread autoReplicaFailoverWorkLoopDelay=10000 autoReplicaFailoverWaitAfterExpiration=30000 autoReplicaFailoverBadNodeExpiration=60000
   [junit4]   2> 322300 T1422 oasc.OverseerCollectionProcessor.run Process current queue of collection creations
   [junit4]   2> 322301 T1421 oasc.Overseer$ClusterStateUpdater.run Starting to work on the main queue
   [junit4]   2> 322301 T1394 oascc.SolrZkClient.makePath makePath: /clusterstate.json
   [junit4]   2> 322304 T1394 oascc.SolrZkClient.makePath makePath: /aliases.json
   [junit4]   2> 322307 T1394 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 322314 T1394 oasc.CorePropertiesLocator.discover Looking for core definitions underneath /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 1F64C8A296BD6261-001/control-001/cores
   [junit4]   2> 322318 T1394 oasc.CorePropertiesLocator.discoverUnder Found core collection1 in /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 1F64C8A296BD6261-001/control-001/cores/collection1/
   [junit4]   2> 322318 T1394 oasc.CorePropertiesLocator.discover Found 1 core definitions
   [junit4]   2> 322320 T1424 oasc.ZkController.publish publishing core=collection1 state=down collection=control_collection
   [junit4]   2> 322320 T1424 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 322322 T1420 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 322322 T1424 oasc.ZkController.waitForCoreNodeName look for our core node name
   [junit4]   2> 322324 T1421 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52981",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:52981_"} current state version: 0
   [junit4]   2> 322325 T1421 oasco.ReplicaMutator.updateState Update state numShards=1 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52981",
   [junit4]   2> 	  "shard":null,
   [junit4]   2> 	  "state":"down",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"1",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:52981_"}
   [junit4]   2> 322326 T1421 oasco.ClusterStateMutator.createCollection building a new cName: control_collection
   [junit4]   2> 322326 T1421 oasco.ReplicaMutator.updateState Assigning new node to shard shard=shard1
   [junit4]   2> 322328 T1420 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 323323 T1424 oasc.ZkController.waitForShardId waiting to find shard id in clusterstate for collection1
   [junit4]   2> 323324 T1424 oasc.ZkController.createCollectionZkNode Check for collection zkNode:control_collection
   [junit4]   2> 323325 T1424 oasc.ZkController.createCollectionZkNode Collection zkNode exists
   [junit4]   2> 323325 T1424 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 323326 T1424 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 323326 T1424 oasc.SolrResourceLoader.<init> new SolrResourceLoader for directory: '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 1F64C8A296BD6261-001/control-001/cores/collection1/'
   [junit4]   2> 323347 T1424 oasc.ZkController.watchZKConfDir watch zkdir /configs/conf1
   [junit4]   2> 323350 T1424 oasc.Config.<init> loaded config solrconfig.xml with version 0 
   [junit4]   2> 323365 T1424 oasc.SolrConfig.refreshRequestParams current version of requestparams : -1
   [junit4]   2> 323389 T1424 oasc.SolrConfig.<init> Using Lucene MatchVersion: 5.1.0
   [junit4]   2> 323455 T1424 oasc.SolrConfig.<init> Loaded SolrConfig: solrconfig.xml
   [junit4]   2> 323457 T1424 oass.IndexSchema.readSchema Reading Solr Schema from /configs/conf1/schema.xml
   [junit4]   2> 323469 T1424 oass.IndexSchema.readSchema [collection1] Schema name=test
   [junit4]   2> 324017 T1424 oass.OpenExchangeRatesOrgProvider.init Initialized with rates=open-exchange-rates.json, refreshInterval=1440.
   [junit4]   2> 324029 T1424 oass.IndexSchema.readSchema default search field in schema is text
   [junit4]   2> 324033 T1424 oass.IndexSchema.readSchema unique key field: id
   [junit4]   2> 324047 T1424 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 324053 T1424 oass.FileExchangeRateProvider.reload Reloading exchange rates from file currency.xml
   [junit4]   2> 324058 T1424 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 324059 T1424 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 324059 T1424 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 324060 T1424 oass.OpenExchangeRatesOrgProvider.reload Reloading exchange rates from open-exchange-rates.json
   [junit4]   2> 324061 T1424 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Unknown key IMPORTANT NOTE
   [junit4]   2> 324061 T1424 oass.OpenExchangeRatesOrgProvider$OpenExchangeRates.<init> WARN Expected key, got STRING
   [junit4]   2> 324061 T1424 oasc.CoreContainer.create Creating SolrCore 'collection1' using configuration from collection control_collection
   [junit4]   2> 324061 T1424 oasc.SolrCore.initDirectoryFactory org.apache.solr.core.MockDirectoryFactory
   [junit4]   2> 324062 T1424 oasc.SolrCore.<init> [collection1] Opening new SolrCore at /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 1F64C8A296BD6261-001/control-001/cores/collection1/, dataDir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 1F64C8A296BD6261-001/tempDir-001/control/data/
   [junit4]   2> 324062 T1424 oasc.JmxMonitoredMap.<init> JMX monitoring is enabled. Adding Solr mbeans to JMX Server: com.sun.jmx.mbeanserver.JmxMBeanServer@3b71b63e
   [junit4]   2> 324063 T1424 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 1F64C8A296BD6261-001/tempDir-001/control/data
   [junit4]   2> 324063 T1424 oasc.SolrCore.getNewIndexDir New index directory detected: old=null new=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 1F64C8A296BD6261-001/tempDir-001/control/data/index/
   [junit4]   2> 324064 T1424 oasc.SolrCore.initIndex WARN [collection1] Solr index directory '/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 1F64C8A296BD6261-001/tempDir-001/control/data/index' doesn't exist. Creating new index...
   [junit4]   2> 324064 T1424 oasc.CachingDirectoryFactory.get return new directory for /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3/temp/solr.cloud.HttpPartitionTest 1F64C8A296BD6261-001/tempDir-001/control/data/index
   [junit4]   2> 324065 T1424 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.AlcoholicMergePolicy: [AlcoholicMergePolicy: minMergeSize=0, mergeFactor=10, maxMergeSize=1260989183, maxMergeSizeForForcedMerge=9223372036854775807, calibrateSizeByDeletes=true, maxMergeDocs=2147483647, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.1]
   [junit4]   2> 324066 T1424 oasc.SolrDeletionPolicy.onCommit SolrDeletionPolicy.onCommit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@7e1b4df3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@329acd33),segFN=segments_1,generation=1}
   [junit4]   2> 324067 T1424 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 324072 T1424 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "nodistrib"
   [junit4]   2> 324072 T1424 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "dedupe"
   [junit4]   2> 324073 T1424 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "dedupe"
   [junit4]   2> 324073 T1424 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "stored_sig"
   [junit4]   2> 324073 T1424 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "stored_sig"
   [junit4]   2> 324074 T1424 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-explicit"
   [junit4]   2> 324074 T1424 oasup.UpdateRequestProcessorChain.init creating updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 324075 T1424 oasup.UpdateRequestProcessorChain.init inserting DistributedUpdateProcessorFactory into updateRequestProcessorChain "distrib-dup-test-chain-implicit"
   [junit4]   2> 324075 T1424 oasc.SolrCore.loadUpdateProcessorChains no updateRequestProcessorChain defined as default, creating implicit default
   [junit4]   2> 324076 T1424 oasc.RequestHandlers.initHandlersFromConfig created /update: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 324076 T1424 oasc.RequestHandlers.initHandlersFromConfig created /update/json: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 324077 T1424 oasc.RequestHandlers.initHandlersFromConfig created /update/csv: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 324077 T1424 oasc.RequestHandlers.initHandlersFromConfig created /update/json/docs: org.apache.solr.handler.UpdateRequestHandler
   [junit4]   2> 324078 T1424 oasc.RequestHandlers.initHandlersFromConfig created /config: org.apache.solr.handler.SolrConfigHandler
   [junit4]   2> 324078 T1424 oasc.RequestHandlers.initHandlersFromConfig created /schema: org.apache.solr.handler.SchemaHandler
   [junit4]   2> 324079 T1424 oasc.RequestHandlers.initHandlersFromConfig created /replication: org.apache.solr.handler.ReplicationHandler
   [junit4]   2> 324079 T1424 oasc.RequestHandlers.initHandlersFromConfig created /get: org.apache.solr.handler.RealTimeGetHandler
   [junit4]   2> 324080 T1424 oasc.RequestHandlers.initHandlersFromConfig created /admin/luke: org.apache.solr.handler.admin.LukeRequestHandler
   [junit4]   2> 324082 T1424 oasc.RequestHandlers.initHandlersFromConfig created /admin/system: org.apache.solr.handler.admin.SystemInfoHandler
   [junit4]   2> 324082 T1424 oasc.RequestHandlers.initHandlersFromConfig created /admin/mbeans: org.apache.solr.handler.admin.SolrInfoMBeanHandler
   [junit4]   2> 324083 T1424 oasc.RequestHandlers.initHandlersFromConfig created /admin/plugins: org.apache.solr.handler.admin.PluginInfoHandler
   [junit4]   2> 324084 T1424 oasc.RequestHandlers.initHandlersFromConfig created /admin/threads: org.apache.solr.handler.admin.ThreadDumpHandler
   [junit4]   2> 324084 T1424 oasc.RequestHandlers.initHandlersFromConfig created /admin/properties: org.apache.solr.handler.admin.PropertiesRequestHandler
   [junit4]   2> 324085 T1424 oasc.RequestHandlers.initHandlersFromConfig created /admin/logging: org.apache.solr.handler.admin.LoggingHandler
   [junit4]   2> 324085 T1424 oasc.RequestHandlers.initHandlersFromConfig created /admin/file: solr.admin.ShowFileRequestHandler
   [junit4]   2> 324086 T1424 oasc.RequestHandlers.initHandlersFromConfig created /admin/ping: org.apache.solr.handler.PingRequestHandler
   [junit4]   2> 324087 T1424 oasc.RequestHandlers.initHandlersFromConfig created standard: solr.StandardRequestHandler
   [junit4]   2> 324111 T1424 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 324114 T1424 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 324116 T1424 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 324118 T1424 oashl.XMLLoader.init xsltCacheLifetimeSeconds=60
   [junit4]   2> 324121 T1424 oasc.SolrCore.initStatsCache Using default statsCache cache: org.apache.solr.search.stats.LocalStatsCache
   [junit4]   2> 324123 T1424 oasu.UpdateHandler.<init> Using UpdateLog implementation: org.apache.solr.update.UpdateLog
   [junit4]   2> 324126 T1424 oasu.CommitTracker.<init> Hard AutoCommit: disabled
   [junit4]   2> 324126 T1424 oasu.CommitTracker.<init> Soft AutoCommit: disabled
   [junit4]   2> 324127 T1424 oasu.RandomMergePolicy.<init> RandomMergePolicy wrapping class org.apache.lucene.index.TieredMergePolicy: [TieredMergePolicy: maxMergeAtOnce=31, maxMergeAtOnceExplicit=30, maxMergedSegmentMB=45.2333984375, floorSegmentMB=0.7060546875, forceMergeDeletesPctAllowed=12.106354893270243, segmentsPerTier=17.0, maxCFSSegmentSizeMB=8.796093022207999E12, noCFSRatio=0.7992942819333042
   [junit4]   2> 324128 T1424 oasc.SolrDeletionPolicy.onInit SolrDeletionPolicy.onInit: commits: num=1
   [junit4]   2> 		commit{dir=RawDirectoryWrapper(RAMDirectory@7e1b4df3 lockFactory=org.apache.lucene.store.SingleInstanceLockFactory@329acd33),segFN=segments_1,generation=1}
   [junit4]   2> 324128 T1424 oasc.SolrDeletionPolicy.updateCommits newest commit generation = 1
   [junit4]   2> 324129 T1424 oass.SolrIndexSearcher.<init> Opening Searcher@7ef3e8db[collection1] main
   [junit4]   2> 324129 T1424 oascc.ZkStateReader.readConfigName Load collection config from:/collections/control_collection
   [junit4]   2> 324130 T1424 oascc.ZkStateReader.readConfigName path=/collections/control_collection configName=conf1 specified config exists in ZooKeeper
   [junit4]   2> 324130 T1424 oasr.ManagedResourceStorage.newStorageIO Setting up ZooKeeper-based storage for the RestManager with znodeBase: /configs/conf1
   [junit4]   2> 324131 T1424 oasr.ManagedResourceStorage$ZooKeeperStorageIO.configure Configured ZooKeeperStorageIO with znodeBase: /configs/conf1
   [junit4]   2> 324131 T1424 oasr.RestManager.init Initializing RestManager with initArgs: {}
   [junit4]   2> 324131 T1424 oasr.ManagedResourceStorage.load Reading _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 324132 T1424 oasr.ManagedResourceStorage$ZooKeeperStorageIO.openInputStream No data found for znode /configs/conf1/_rest_managed.json
   [junit4]   2> 324133 T1424 oasr.ManagedResourceStorage.load Loaded null at path _rest_managed.json using ZooKeeperStorageIO:path=/configs/conf1
   [junit4]   2> 324133 T1424 oasr.RestManager.init Initializing 0 registered ManagedResources
   [junit4]   2> 324133 T1424 oash.ReplicationHandler.inform Commits will be reserved for  10000
   [junit4]   2> 324134 T1425 oasc.SolrCore.registerSearcher [collection1] Registered new searcher Searcher@7ef3e8db[collection1] main{ExitableDirectoryReader(UninvertingDirectoryReader())}
   [junit4]   2> 324135 T1424 oasc.CoreContainer.registerCore registering core: collection1
   [junit4]   2> 324136 T1428 oasc.ZkController.register Register replica - core:collection1 address:http://127.0.0.1:52981 collection:control_collection shard:shard1
   [junit4]   2> 324136 T1394 oass.SolrDispatchFilter.init user.dir=/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J3
   [junit4]   2> 324137 T1394 oass.SolrDispatchFilter.init SolrDispatchFilter.init() done
   [junit4]   2> 324137 T1428 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leader_elect/shard1/election
   [junit4]   2> 324142 T1394 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 324143 T1428 oasc.ShardLeaderElectionContext.runLeaderProcess Running the leader process for shard shard1
   [junit4]   2> 324143 T1394 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 324145 T1420 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 324145 T1432 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@2c86725d name:ZooKeeperConnection Watcher:127.0.0.1:52978/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 324145 T1428 oasc.ShardLeaderElectionContext.waitForReplicasToComeUp Enough replicas found to continue.
   [junit4]   2> 324146 T1394 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 324146 T1428 oasc.ShardLeaderElectionContext.runLeaderProcess I may be the new leader - try and sync
   [junit4]   2> ASYNC  NEW_CORE C127 name=collection1 org.apache.solr.core.SolrCore@66d4c83f url=http://127.0.0.1:52981/collection1 node=127.0.0.1:52981_ C127_STATE=coll:control_collection core:collection1 props:{base_url=http://127.0.0.1:52981, state=down, core=collection1, node_name=127.0.0.1:52981_}
   [junit4]   2> 324146 T1428 C127 P52981 oasc.SyncStrategy.sync Sync replicas to http://127.0.0.1:52981/collection1/
   [junit4]   2> 324147 T1394 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 324147 T1421 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection"} current state version: 1
   [junit4]   2> 324147 T1428 C127 P52981 oasc.SyncStrategy.syncReplicas Sync Success - now sync replicas to me
   [junit4]   2> 324148 T1428 C127 P52981 oasc.SyncStrategy.syncToMe http://127.0.0.1:52981/collection1/ has no replicas
   [junit4]   2> 324148 T1428 oasc.ShardLeaderElectionContext.runLeaderProcess I am the new leader: http://127.0.0.1:52981/collection1/ shard1
   [junit4]   2> 324149 T1428 oascc.SolrZkClient.makePath makePath: /collections/control_collection/leaders/shard1
   [junit4]   2> 324149 T1394 oascc.ZkStateReader.createClusterStateWatchersAndUpdate Updating cluster state from ZooKeeper... 
   [junit4]   2> 324150 T1420 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 324153 T1394 oasc.ChaosMonkey.monkeyLog monkey: init - expire sessions:false cause connection loss:false
   [junit4]   2> 324153 T1394 oasc.AbstractFullDistribZkTestBase.createJettys Creating collection1 with stateFormat=2
   [junit4]   2> 324154 T1394 oascc.SolrZkClient.createZkCredentialsToAddAutomatically Using default ZkCredentialsProvider
   [junit4]   2> 324155 T1420 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 324155 T1394 oascc.ConnectionManager.waitForConnected Waiting for client to connect to ZooKeeper
   [junit4]   2> 324157 T1421 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"leader",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52981",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "state":"active"} current state version: 2
   [junit4]   2> 324157 T1435 oascc.ConnectionManager.process Watcher org.apache.solr.common.cloud.ConnectionManager@4d49f152 name:ZooKeeperConnection Watcher:127.0.0.1:52978/solr got event WatchedEvent state:SyncConnected type:None path:null path:null type:None
   [junit4]   2> 324157 T1394 oascc.ConnectionManager.waitForConnected Client is connected to ZooKeeper
   [junit4]   2> 324158 T1394 oascc.SolrZkClient.createZkACLProvider Using default ZkACLProvider
   [junit4]   2> 324161 T1420 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 324163 T1421 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"create",
   [junit4]   2> 	  "name":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "stateFormat":"2"} current state version: 2
   [junit4]   2> 324164 T1421 oasco.ClusterStateMutator.createCollection building a new cName: collection1
   [junit4]   2> 324165 T1432 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 324165 T1420 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:NodeDataChanged path:/clusterstate.json, has occurred - updating... (live nodes size: 1)
   [junit4]   2> 324206 T1428 oasc.ZkController.register We are http://127.0.0.1:52981/collection1/ and leader is http://127.0.0.1:52981/collection1/
   [junit4]   2> 324207 T1428 oasc.ZkController.register No LogReplay needed for core=collection1 baseURL=http://127.0.0.1:52981
   [junit4]   2> 324207 T1428 oasc.ZkController.checkRecovery I am the leader, no recovery necessary
   [junit4]   2> 324208 T1428 oasc.ZkController.publish publishing core=collection1 state=active collection=control_collection
   [junit4]   2> 324208 T1428 oasc.ZkController.publish numShards not found on descriptor - reading it from system property
   [junit4]   2> 324210 T1420 oasc.DistributedQueue$LatchWatcher.process NodeChildrenChanged fired on path /overseer/queue state SyncConnected
   [junit4]   2> 324211 T1421 oasc.Overseer$ClusterStateUpdater.run processMessage: queueSize: 1, message = {
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52981",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:52981_"} current state version: 3
   [junit4]   2> 324212 T1421 oasco.ReplicaMutator.updateState Update state numShards=2 message={
   [junit4]   2> 	  "operation":"state",
   [junit4]   2> 	  "base_url":"http://127.0.0.1:52981",
   [junit4]   2> 	  "shard":"shard1",
   [junit4]   2> 	  "state":"active",
   [junit4]   2> 	  "collection":"control_collection",
   [junit4]   2> 	  "core_node_name":"core_node1",
   [junit4]   2> 	  "core":"collection1",
   [junit4]   2> 	  "numShards":"2",
   [junit4]   2> 	  "roles":null,
   [junit4]   2> 	  "node_name":"127.0.0.1:52981_"}
   [junit4]   2> 324213 T1421 oasco.ZkStateWriter.writePendingUpdates going to create_collection /collections/collection1/state.json
   [junit4]   2> 324215 T1420 oascc.ZkStateReader$2.process A cluster state change: WatchedEvent state:SyncConnected type:No

[...truncated too long message...]

114 oasc.SocketProxy.close WARN Closing 9 connections to: http://127.0.0.1:37433/kcd/zh
   [junit4]   2> 2298778 T5114 oasc.ReplicationFactorTest.distribTearDown super.distribTearDown complete, closing all socket proxies
   [junit4]   2> 2298778 T5114 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37494/kcd/zh
   [junit4]   2> 2298779 T5114 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37445/kcd/zh
   [junit4]   2> 2298779 T5114 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37470/kcd/zh
   [junit4]   2> 2298779 T5114 oasc.SocketProxy.close WARN Closing 0 connections to: http://127.0.0.1:37433/kcd/zh
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=ReplicationFactorTest -Dtests.method=test -Dtests.seed=1F64C8A296BD6261 -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=es -Dtests.timezone=Asia/Beirut -Dtests.asserts=true -Dtests.file.encoding=UTF-8
   [junit4] ERROR   75.0s J2 | ReplicationFactorTest.test <<<
   [junit4]    > Throwable #1: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37433/kcd/zh/repfacttest_c8n_1x3_shard1_replica2
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([1F64C8A296BD6261:9730F77838410F99]:0)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:597)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.sendRequest(CloudSolrClient.java:918)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.requestWithRetryOnStaleState(CloudSolrClient.java:809)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.request(CloudSolrClient.java:752)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.sendDoc(ReplicationFactorTest.java:319)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.testRf3(ReplicationFactorTest.java:233)
   [junit4]    > 	at org.apache.solr.cloud.ReplicationFactorTest.test(ReplicationFactorTest.java:112)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsFixedStatement.callStatement(BaseDistributedSearchTestCase.java:945)
   [junit4]    > 	at org.apache.solr.BaseDistributedSearchTestCase$ShardsRepeatRule$ShardsStatement.evaluate(BaseDistributedSearchTestCase.java:920)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]    > Caused by: org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://127.0.0.1:37433/kcd/zh/repfacttest_c8n_1x3_shard1_replica2
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:572)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:214)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.request(HttpSolrClient.java:210)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.doRequest(LBHttpSolrClient.java:370)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.LBHttpSolrClient.request(LBHttpSolrClient.java:325)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.CloudSolrClient.directUpdate(CloudSolrClient.java:594)
   [junit4]    > 	... 48 more
   [junit4]    > Caused by: org.apache.http.NoHttpResponseException: The target server failed to respond
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:143)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
   [junit4]    > 	at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)
   [junit4]    > 	at org.apache.http.impl.AbstractHttpClientConnection.receiveResponseHeader(AbstractHttpClientConnection.java:283)
   [junit4]    > 	at org.apache.http.impl.conn.DefaultClientConnection.receiveResponseHeader(DefaultClientConnection.java:251)
   [junit4]    > 	at org.apache.http.impl.conn.ManagedClientConnectionImpl.receiveResponseHeader(ManagedClientConnectionImpl.java:197)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)
   [junit4]    > 	at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:682)
   [junit4]    > 	at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:486)
   [junit4]    > 	at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
   [junit4]    > 	at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
   [junit4]    > 	at org.apache.solr.client.solrj.impl.HttpSolrClient.executeMethod(HttpSolrClient.java:464)
   [junit4]    > 	... 53 more
   [junit4]   2> 2298801 T5114 oas.SolrTestCaseJ4.deleteCore ###deleteCore
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build/solr-core/test/J2/temp/solr.cloud.ReplicationFactorTest 1F64C8A296BD6261-001
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {_version_=Lucene50(blocksize=128), id=FST50, intDefault=Lucene50(blocksize=128), timestamp=PostingsFormat(name=Direct), multiDefault=PostingsFormat(name=Direct), text=PostingsFormat(name=MockRandom), a_t=PostingsFormat(name=Direct), range_facet_l=PostingsFormat(name=Direct)}, docValues:{timestamp=DocValuesFormat(name=Memory)}, sim=DefaultSimilarity, locale=es, timezone=Asia/Beirut
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=177073280,total=486539264
   [junit4]   2> NOTE: All tests run in this JVM: [ChangedSchemaMergeTest, SimpleMLTQParserTest, DistributedFacetPivotWhiteBoxTest, StandardRequestHandlerTest, TestExtendedDismaxParser, URLClassifyProcessorTest, TestQuerySenderNoQuery, SuggestComponentTest, MoreLikeThisHandlerTest, DirectUpdateHandlerOptimizeTest, RAMDirectoryFactoryTest, TestCloudPivotFacet, TestRTGBase, SynonymTokenizerTest, TestSuggestSpellingConverter, SOLR749Test, RankQueryTest, TestClusterStateMutator, LukeRequestHandlerTest, TestAnalyzeInfixSuggestions, CacheHeaderTest, TestLuceneMatchVersion, FacetPivotSmallTest, TestReplicationHandlerBackup, IgnoreCommitOptimizeUpdateProcessorFactoryTest, AssignTest, TestHashPartitioner, TestReload, SolrIndexSplitterTest, CursorPagingTest, TestPerFieldSimilarity, TestComponentsName, CoreMergeIndexesAdminHandlerTest, TestRealTimeGet, TestRemoteStreaming, TestLazyCores, DistributedSuggestComponentTest, TestFastWriter, TestReplicationHandler, FileBasedSpellCheckerTest, TestHighFrequencyDictionaryFactory, SpellCheckComponentTest, TestDistributedSearch, DocumentBuilderTest, TestShardHandlerFactory, TestRandomDVFaceting, TestConfigReload, OverseerStatusTest, SolrXmlInZkTest, TestAnalyzedSuggestions, TestSolr4Spatial2, TestLRUCache, TestInfoStreamLogging, DirectUpdateHandlerTest, TestMergePolicyConfig, FullSolrCloudDistribCmdsTest, TestSort, HighlighterTest, SimplePostToolTest, SolrRequestParserTest, NoCacheHeaderTest, SolrCoreCheckLockOnStartupTest, SortByFunctionTest, XsltUpdateRequestHandlerTest, TestQueryUtils, PrimitiveFieldTypeTest, RequiredFieldsTest, SolrPluginUtilsTest, TestCSVResponseWriter, TestPHPSerializedResponseWriter, CopyFieldTest, MultiTermTest, TestDocSet, TestSearchPerf, TestElisionMultitermQuery, TestFuzzyAnalyzedSuggestions, TestCharFilters, EchoParamsTest, TestIBSimilarityFactory, TestDefaultSimilarityFactory, ChaosMonkeyNothingIsSafeTest, DeleteShardTest, DistribCursorPagingTest, MultiThreadedOCPTest, ReplicationFactorTest]
   [junit4] Completed on J2 in 74.98s, 1 test, 1 error <<< FAILURES!

[...truncated 310 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/build.xml:191: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/solr/common-build.xml:509: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 468 suites, 1870 tests, 7 errors, 58 ignored (9 assumptions)

Total time: 148 minutes 36 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 784841887 bytes
Compression is 0.0%
Took 2 min 23 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure



Re: [JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 770 - Still Failing

Posted by Robert Muir <rc...@gmail.com>.
I committed a fix. I don't like this backdoor here. If other jenkins
servers add other paths, they will need to be added too.

we should not overengineer this, but keep it underengineered and discourage it.

On Thu, Feb 26, 2015 at 9:36 AM, Apache Jenkins Server
<je...@builds.apache.org> wrote:
> Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/770/
>
> 35 tests failed.
> REGRESSION:  org.apache.lucene.analysis.TestGraphTokenizers.testDoubleMockGraphTokenFilterRandom
>
> Error Message:
> access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>
> Stack Trace:
> java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>         at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:F5F5B75438472D0B]:0)
>         at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
>         at java.security.AccessController.checkPermission(AccessController.java:559)
>         at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
>         at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
>         at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
>         at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
>         at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
>         at java.nio.file.Files.readAttributes(Files.java:1686)
>         at java.nio.file.Files.size(Files.java:2275)
>         at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:549)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:509)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:433)
>         at org.apache.lucene.analysis.TestGraphTokenizers.testDoubleMockGraphTokenFilterRandom(TestGraphTokenizers.java:342)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
> REGRESSION:  org.apache.lucene.analysis.TestGraphTokenizers.testMockGraphTokenFilterAfterHolesRandom
>
> Error Message:
> access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>
> Stack Trace:
> java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>         at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:9FA2F597CB33FAD0]:0)
>         at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
>         at java.security.AccessController.checkPermission(AccessController.java:559)
>         at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
>         at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
>         at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
>         at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
>         at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
>         at java.nio.file.Files.readAttributes(Files.java:1686)
>         at java.nio.file.Files.size(Files.java:2275)
>         at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:549)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:509)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:433)
>         at org.apache.lucene.analysis.TestGraphTokenizers.testMockGraphTokenFilterAfterHolesRandom(TestGraphTokenizers.java:390)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
> REGRESSION:  org.apache.lucene.analysis.TestGraphTokenizers.testMockGraphTokenFilterRandom
>
> Error Message:
> access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>
> Stack Trace:
> java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>         at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:DAA80F729969AB78]:0)
>         at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
>         at java.security.AccessController.checkPermission(AccessController.java:559)
>         at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
>         at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
>         at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
>         at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
>         at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
>         at java.nio.file.Files.readAttributes(Files.java:1686)
>         at java.nio.file.Files.size(Files.java:2275)
>         at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:549)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:509)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:433)
>         at org.apache.lucene.analysis.TestGraphTokenizers.testMockGraphTokenFilterRandom(TestGraphTokenizers.java:317)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
> REGRESSION:  org.apache.lucene.analysis.TestGraphTokenizers.testMockGraphTokenFilterBeforeHolesRandom
>
> Error Message:
> access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>
> Stack Trace:
> java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>         at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:25F5C12789A857AB]:0)
>         at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
>         at java.security.AccessController.checkPermission(AccessController.java:559)
>         at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
>         at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
>         at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
>         at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
>         at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
>         at java.nio.file.Files.readAttributes(Files.java:1686)
>         at java.nio.file.Files.size(Files.java:2275)
>         at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:549)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:509)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:433)
>         at org.apache.lucene.analysis.TestGraphTokenizers.testMockGraphTokenFilterBeforeHolesRandom(TestGraphTokenizers.java:366)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
> REGRESSION:  org.apache.lucene.analysis.TestLookaheadTokenFilter.testRandomStrings
>
> Error Message:
> access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>
> Stack Trace:
> java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>         at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:F63FA727BBC061EF]:0)
>         at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
>         at java.security.AccessController.checkPermission(AccessController.java:559)
>         at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
>         at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
>         at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
>         at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
>         at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
>         at java.nio.file.Files.readAttributes(Files.java:1686)
>         at java.nio.file.Files.size(Files.java:2275)
>         at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:549)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:509)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:433)
>         at org.apache.lucene.analysis.TestLookaheadTokenFilter.testRandomStrings(TestLookaheadTokenFilter.java:37)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
> REGRESSION:  org.apache.lucene.analysis.TestLookaheadTokenFilter.testNeverCallingPeek
>
> Error Message:
> access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>
> Stack Trace:
> java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>         at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:8D5310FEFA976225]:0)
>         at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
>         at java.security.AccessController.checkPermission(AccessController.java:559)
>         at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
>         at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
>         at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
>         at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
>         at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
>         at java.nio.file.Files.readAttributes(Files.java:1686)
>         at java.nio.file.Files.size(Files.java:2275)
>         at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:549)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:509)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:433)
>         at org.apache.lucene.analysis.TestLookaheadTokenFilter.testNeverCallingPeek(TestLookaheadTokenFilter.java:66)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
> REGRESSION:  org.apache.lucene.analysis.TestMockAnalyzer.testRandomRegexps
>
> Error Message:
> access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>
> Stack Trace:
> java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>         at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:9D624909112F86C9]:0)
>         at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
>         at java.security.AccessController.checkPermission(AccessController.java:559)
>         at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
>         at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
>         at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
>         at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
>         at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
>         at java.nio.file.Files.readAttributes(Files.java:1686)
>         at java.nio.file.Files.size(Files.java:2275)
>         at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:549)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:509)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:428)
>         at org.apache.lucene.analysis.TestMockAnalyzer.testRandomRegexps(TestMockAnalyzer.java:245)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
> REGRESSION:  org.apache.lucene.analysis.TestMockAnalyzer.testTwoChars
>
> Error Message:
> access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>
> Stack Trace:
> java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>         at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:983F3607E55D8D1F]:0)
>         at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
>         at java.security.AccessController.checkPermission(AccessController.java:559)
>         at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
>         at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
>         at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
>         at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
>         at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
>         at java.nio.file.Files.readAttributes(Files.java:1686)
>         at java.nio.file.Files.size(Files.java:2275)
>         at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:549)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:509)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:428)
>         at org.apache.lucene.analysis.TestMockAnalyzer.testTwoChars(TestMockAnalyzer.java:115)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
> REGRESSION:  org.apache.lucene.analysis.TestMockAnalyzer.testThreeChars
>
> Error Message:
> access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>
> Stack Trace:
> java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>         at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:CA5498F98659D9E]:0)
>         at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
>         at java.security.AccessController.checkPermission(AccessController.java:559)
>         at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
>         at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
>         at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
>         at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
>         at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
>         at java.nio.file.Files.readAttributes(Files.java:1686)
>         at java.nio.file.Files.size(Files.java:2275)
>         at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:549)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:509)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:428)
>         at org.apache.lucene.analysis.TestMockAnalyzer.testThreeChars(TestMockAnalyzer.java:136)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
> REGRESSION:  org.apache.lucene.analysis.TestMockAnalyzer.testUppercase
>
> Error Message:
> access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>
> Stack Trace:
> java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>         at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:9982189CA83F33D2]:0)
>         at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
>         at java.security.AccessController.checkPermission(AccessController.java:559)
>         at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
>         at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
>         at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
>         at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
>         at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
>         at java.nio.file.Files.readAttributes(Files.java:1686)
>         at java.nio.file.Files.size(Files.java:2275)
>         at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:549)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:509)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:428)
>         at org.apache.lucene.analysis.TestMockAnalyzer.testUppercase(TestMockAnalyzer.java:154)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
> REGRESSION:  org.apache.lucene.analysis.TestMockAnalyzer.testRandomStrings
>
> Error Message:
> access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>
> Stack Trace:
> java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>         at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:F63FA727BBC061EF]:0)
>         at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
>         at java.security.AccessController.checkPermission(AccessController.java:559)
>         at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
>         at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
>         at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
>         at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
>         at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
>         at java.nio.file.Files.readAttributes(Files.java:1686)
>         at java.nio.file.Files.size(Files.java:2275)
>         at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:549)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:509)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:428)
>         at org.apache.lucene.analysis.TestMockAnalyzer.testRandomStrings(TestMockAnalyzer.java:228)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
> REGRESSION:  org.apache.lucene.analysis.TestMockAnalyzer.testSingleChar
>
> Error Message:
> access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>
> Stack Trace:
> java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>         at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:F5153C673E9BD106]:0)
>         at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
>         at java.security.AccessController.checkPermission(AccessController.java:559)
>         at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
>         at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
>         at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
>         at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
>         at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
>         at java.nio.file.Files.readAttributes(Files.java:1686)
>         at java.nio.file.Files.size(Files.java:2275)
>         at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:549)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:509)
>         at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:428)
>         at org.apache.lucene.analysis.TestMockAnalyzer.testSingleChar(TestMockAnalyzer.java:94)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
> REGRESSION:  org.apache.lucene.codecs.asserting.TestAssertingDocValuesFormat.testInvertedWrite
>
> Error Message:
> access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>
> Stack Trace:
> java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>         at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:67C2EA52669FBD06]:0)
>         at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
>         at java.security.AccessController.checkPermission(AccessController.java:559)
>         at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
>         at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
>         at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
>         at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
>         at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
>         at java.nio.file.Files.readAttributes(Files.java:1686)
>         at java.nio.file.Files.size(Files.java:2275)
>         at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
>         at org.apache.lucene.index.BasePostingsFormatTestCase.testInvertedWrite(BasePostingsFormatTestCase.java:1736)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
> REGRESSION:  org.apache.lucene.codecs.asserting.TestAssertingPostingsFormat.testInvertedWrite
>
> Error Message:
> access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>
> Stack Trace:
> java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>         at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:67C2EA52669FBD06]:0)
>         at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
>         at java.security.AccessController.checkPermission(AccessController.java:559)
>         at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
>         at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
>         at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
>         at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
>         at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
>         at java.nio.file.Files.readAttributes(Files.java:1686)
>         at java.nio.file.Files.size(Files.java:2275)
>         at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
>         at org.apache.lucene.index.BasePostingsFormatTestCase.testInvertedWrite(BasePostingsFormatTestCase.java:1736)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
> REGRESSION:  org.apache.lucene.codecs.lucene50.TestBlockPostingsFormat.testInvertedWrite
>
> Error Message:
> access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>
> Stack Trace:
> java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>         at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:67C2EA52669FBD06]:0)
>         at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
>         at java.security.AccessController.checkPermission(AccessController.java:559)
>         at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
>         at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
>         at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
>         at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
>         at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
>         at java.nio.file.Files.readAttributes(Files.java:1686)
>         at java.nio.file.Files.size(Files.java:2275)
>         at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
>         at org.apache.lucene.index.BasePostingsFormatTestCase.testInvertedWrite(BasePostingsFormatTestCase.java:1736)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
> REGRESSION:  org.apache.lucene.codecs.perfield.TestPerFieldPostingsFormat.testInvertedWrite
>
> Error Message:
> access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>
> Stack Trace:
> java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>         at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:67C2EA52669FBD06]:0)
>         at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
>         at java.security.AccessController.checkPermission(AccessController.java:559)
>         at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
>         at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
>         at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
>         at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
>         at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
>         at java.nio.file.Files.readAttributes(Files.java:1686)
>         at java.nio.file.Files.size(Files.java:2275)
>         at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
>         at org.apache.lucene.index.BasePostingsFormatTestCase.testInvertedWrite(BasePostingsFormatTestCase.java:1736)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
> REGRESSION:  org.apache.lucene.index.TestCustomNorms.testFloatNorms
>
> Error Message:
> access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>
> Stack Trace:
> java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>         at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:3796442599ECB78E]:0)
>         at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
>         at java.security.AccessController.checkPermission(AccessController.java:559)
>         at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
>         at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
>         at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
>         at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
>         at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
>         at java.nio.file.Files.readAttributes(Files.java:1686)
>         at java.nio.file.Files.size(Files.java:2275)
>         at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
>         at org.apache.lucene.index.TestCustomNorms.testFloatNorms(TestCustomNorms.java:52)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
>         at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
>         at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
>         at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
>         at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
>         at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
>         at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
>         at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
>         at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
>         at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
>         at java.lang.Thread.run(Thread.java:745)
>
>
> REGRESSION:  org.apache.lucene.index.TestDuelingCodecs.testEquals
>
> Error Message:
> access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>
> Stack Trace:
> java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>         at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:7AD9F402D6FD66D8]:0)
>         at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
>         at java.security.AccessController.checkPermission(AccessController.java:559)
>         at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
>         at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
>         at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
>         at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
>         at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
>         at java.nio.file.Files.readAttributes(Files.java:1686)
>         at java.nio.file.Files.size(Files.java:2275)
>         at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
>         at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
>         at org.apache.lucene.index.TestDuelingCodecs.createRandomIndex(TestDuelingCodecs.java:111)
>         at org.apache.lucene.index.TestDuelingCodecs.testEquals(TestDuelingCodecs.java:148)
>         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
>         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>         at java.lang.reflect.Method.invoke(Method.java:606)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
>         at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
>         at com.carrotsearch.randomiz
>
> [...truncated too long message...]
>
> s.txt" "read")
>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:67C2EA52669FBD06]:0)
>    [junit4]    >        at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
>    [junit4]    >        at java.security.AccessController.checkPermission(AccessController.java:559)
>    [junit4]    >        at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
>    [junit4]    >        at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
>    [junit4]    >        at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
>    [junit4]    >        at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
>    [junit4]    >        at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
>    [junit4]    >        at java.nio.file.Files.readAttributes(Files.java:1686)
>    [junit4]    >        at java.nio.file.Files.size(Files.java:2275)
>    [junit4]    >        at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
>    [junit4]    >        at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
>    [junit4]    >        at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
>    [junit4]    >        at org.apache.lucene.index.BasePostingsFormatTestCase.testInvertedWrite(BasePostingsFormatTestCase.java:1736)
>    [junit4]    >        at java.lang.Thread.run(Thread.java:745)
>    [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J0/temp/lucene.codecs.lucene50.TestBlockPostingsFormat 7EB6A79918C436DA-001
>    [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=DefaultSimilarity, locale=en_SG, timezone=Europe/Minsk
>    [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=324570496,total=424673280
>    [junit4]   2> NOTE: All tests run in this JVM: [TestManyFields, TestNorms, TestDocValuesTermsQuery, TestSearchWithThreads, TestCharsRef, TestScorerPerf, TestAssertingTermVectorsFormat, TestSortRescorer, TestDocIdSet, TestSumDocFreq, TestTermVectorsWriter, TestMatchAllDocsQuery, TestNRTThreads, TestOfflineSorter, TestPositionIncrement, TestCharTermAttributeImpl, TestDemo, InBeforeClass, InAfterClass, InTestMethod, NonStringProperties, TestTopFieldCollector, TestPayloadTermQuery, TestCachingTokenFilter, TestNRTCachingDirectory, TestForTooMuchCloning, TestSegmentTermEnum, TestTermsEnum, TestCachingCollector, TestNeedsScores, TestSubScorerFreqs, TestQueryRescorer, TestLongPostings, TestRollingBuffer, TestTryDelete, TestTermdocPerf, TestAutomatonQuery, TestVirtualMethod, Test4GBStoredFields, TestBooleanMinShouldMatch, TestReaderClosed, TestRegexpRandom, TestAssertingDocValuesFormat, TestAttributeSource, TestMathUtil, TestUTF32ToUTF8, TestIndexInput, TestPrefixFilter, TestScoreCachingWrappingScorer, TestMultiValuedNumericRangeQuery, TestSegmentMerger, TestInPlaceMergeSorter, TestRecyclingByteBlockAllocator, TestByteBlockPool, TestIndexWriterMerging, TestBytesStore, TestPackedInts, TestSearcherManager, TestDeletionPolicy, TestBooleanOr, TestBasics, TestFlushByRamOrCountsPolicy, TestSloppyPhraseQuery, TestIndexWriterMergePolicy, TestBoolean2, TestAtomicUpdate, TestIndexWriterCommit, TestNRTReaderWithThreads, TestTieredMergePolicy, TestDirectoryReader, TestCompressingStoredFieldsFormat, TestBlockPostingsFormat]
>    [junit4] Completed on J0 in 18.52s, 18 tests, 1 error <<< FAILURES!
>
> [...truncated 70 lines...]
>    [junit4] Suite: org.apache.lucene.search.TestControlledRealTimeReopenThread
>    [junit4] IGNOR/A 0.01s J2 | TestControlledRealTimeReopenThread.testCRTReopen
>    [junit4]    > Assumption #1: 'awaitsfix' test group is disabled (@AwaitsFix(bugUrl=https://issues.apache.org/jira/browse/LUCENE-5737))
>    [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
>    [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestControlledRealTimeReopenThread -Dtests.method=testControlledRealTimeReopenThread -Dtests.seed=7EB6A79918C436DA -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=pt_BR -Dtests.timezone=America/Grenada -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
>    [junit4] ERROR   0.01s J2 | TestControlledRealTimeReopenThread.testControlledRealTimeReopenThread <<<
>    [junit4]    > Throwable #1: java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
>    [junit4]    >        at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:815FB65AEBC71ED6]:0)
>    [junit4]    >        at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
>    [junit4]    >        at java.security.AccessController.checkPermission(AccessController.java:559)
>    [junit4]    >        at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
>    [junit4]    >        at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
>    [junit4]    >        at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
>    [junit4]    >        at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
>    [junit4]    >        at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
>    [junit4]    >        at java.nio.file.Files.readAttributes(Files.java:1686)
>    [junit4]    >        at java.nio.file.Files.size(Files.java:2275)
>    [junit4]    >        at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
>    [junit4]    >        at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
>    [junit4]    >        at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:71)
>    [junit4]    >        at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase.runTest(ThreadedIndexingAndSearchingTestCase.java:435)
>    [junit4]    >        at org.apache.lucene.search.TestControlledRealTimeReopenThread.testControlledRealTimeReopenThread(TestControlledRealTimeReopenThread.java:70)
>    [junit4]    >        at java.lang.Thread.run(Thread.java:745)
>    [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {test=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {test=IB SPL-LZ(0.3)}, locale=pt_BR, timezone=America/Grenada
>    [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=176860360,total=434634752
>    [junit4]   2> NOTE: All tests run in this JVM: [TestTerms, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, TestSloppyMath, TestDuelingCodecs, TestSimilarityProvider, TestMixedDocValuesUpdates, TestDateSort, TestSearchForDuplicates, TestPrefixInBooleanQuery, TestSortRandom, TestPrefixCodedTerms, TestMultiFields, TestSimilarity2, TestDirectory, Test2BPostings, TestSizeBoundedForceMerge, TestPrefixRandom, TestMinimize, Nested2, Nested, TestFilterLeafReader, TestSpansAdvanced2, TestDeterminism, TestMergeRateLimiter, TestExceedMaxTermLength, TestCodecs, TestIndexWriterLockRelease, TestOmitTf, TestIndexWriterOutOfFileDescriptors, TestToken, TestLock, TestSearchAfter, TestCustomNorms, TestComplexExplanations, TestTermVectors, TestBlockPostingsFormat3, TestTopDocsMerge, TestFileSwitchDirectory, TestFilterCachingPolicy, TestMockCharFilter, TestFilterIterator, TestTermsEnum2, TestRamUsageEstimator, TestCrash, TestParallelTermEnum, Test2BTerms, TestDocInverterPerFieldErrorInfo, TestStressNRT, TestParallelCompositeReader, TestIndexWriterOutOfMemory, TestMinShouldMatch2, TestSegmentReader, TestFastDecompressionMode, TestIndexWriterNRTIsCurrent, TestFieldCacheRewriteMethod, TestFieldsReader, TestCustomSearcherSort, TestDocValuesIndexing, TestNumericUtils, TestDeterminizeLexicon, TestIndexWriterOnDiskFull, TestSpans, TestMixedCodecs, TestTransactionRollback, TestFastCompressionMode, TestLazyProxSkipping, TestRamUsageEstimatorOnWildAnimals, TestMergeSchedulerExternal, TestThreadedForceMerge, TestSloppyPhraseQuery2, TestCodecUtil, TestDocValues, TestExitableDirectoryReader, TestFieldReuse, TestIndexReaderClose, TestIndexWriterThreadsToSegments, TestInfoStream, TestLegacyPostings, TestLogMergePolicy, TestNRTReaderCleanup, TestMockFilesystems, TestControlledRealTimeReopenThread]
>    [junit4] Completed on J2 in 0.07s, 5 tests, 1 error, 1 skipped <<< FAILURES!
>
> [...truncated 162 lines...]
> BUILD FAILED
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:49: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
> /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 418 suites, 3307 tests, 2 suite-level errors, 33 errors, 38 ignored (28 assumptions)
>
> Total time: 31 minutes 40 seconds
> Build step 'Invoke Ant' marked build as failure
> Archiving artifacts
> Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
> Archived 5 artifacts
> Archive block size is 32768
> Received 0 blocks and 4070516 bytes
> Compression is 0.0%
> Took 1.1 sec
> Recording test results
> Email was triggered for: Failure
> Sending email for trigger: Failure
>
>
>
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org


[JENKINS] Lucene-Solr-NightlyTests-5.x - Build # 770 - Still Failing

Posted by Apache Jenkins Server <je...@builds.apache.org>.
Build: https://builds.apache.org/job/Lucene-Solr-NightlyTests-5.x/770/

35 tests failed.
REGRESSION:  org.apache.lucene.analysis.TestGraphTokenizers.testDoubleMockGraphTokenFilterRandom

Error Message:
access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
	at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:F5F5B75438472D0B]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
	at java.security.AccessController.checkPermission(AccessController.java:559)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
	at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
	at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
	at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
	at java.nio.file.Files.readAttributes(Files.java:1686)
	at java.nio.file.Files.size(Files.java:2275)
	at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:549)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:509)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:433)
	at org.apache.lucene.analysis.TestGraphTokenizers.testDoubleMockGraphTokenFilterRandom(TestGraphTokenizers.java:342)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.lucene.analysis.TestGraphTokenizers.testMockGraphTokenFilterAfterHolesRandom

Error Message:
access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
	at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:9FA2F597CB33FAD0]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
	at java.security.AccessController.checkPermission(AccessController.java:559)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
	at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
	at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
	at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
	at java.nio.file.Files.readAttributes(Files.java:1686)
	at java.nio.file.Files.size(Files.java:2275)
	at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:549)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:509)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:433)
	at org.apache.lucene.analysis.TestGraphTokenizers.testMockGraphTokenFilterAfterHolesRandom(TestGraphTokenizers.java:390)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.lucene.analysis.TestGraphTokenizers.testMockGraphTokenFilterRandom

Error Message:
access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
	at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:DAA80F729969AB78]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
	at java.security.AccessController.checkPermission(AccessController.java:559)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
	at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
	at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
	at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
	at java.nio.file.Files.readAttributes(Files.java:1686)
	at java.nio.file.Files.size(Files.java:2275)
	at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:549)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:509)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:433)
	at org.apache.lucene.analysis.TestGraphTokenizers.testMockGraphTokenFilterRandom(TestGraphTokenizers.java:317)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.lucene.analysis.TestGraphTokenizers.testMockGraphTokenFilterBeforeHolesRandom

Error Message:
access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
	at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:25F5C12789A857AB]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
	at java.security.AccessController.checkPermission(AccessController.java:559)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
	at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
	at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
	at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
	at java.nio.file.Files.readAttributes(Files.java:1686)
	at java.nio.file.Files.size(Files.java:2275)
	at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:549)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:509)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:433)
	at org.apache.lucene.analysis.TestGraphTokenizers.testMockGraphTokenFilterBeforeHolesRandom(TestGraphTokenizers.java:366)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.lucene.analysis.TestLookaheadTokenFilter.testRandomStrings

Error Message:
access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
	at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:F63FA727BBC061EF]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
	at java.security.AccessController.checkPermission(AccessController.java:559)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
	at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
	at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
	at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
	at java.nio.file.Files.readAttributes(Files.java:1686)
	at java.nio.file.Files.size(Files.java:2275)
	at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:549)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:509)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:433)
	at org.apache.lucene.analysis.TestLookaheadTokenFilter.testRandomStrings(TestLookaheadTokenFilter.java:37)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.lucene.analysis.TestLookaheadTokenFilter.testNeverCallingPeek

Error Message:
access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
	at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:8D5310FEFA976225]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
	at java.security.AccessController.checkPermission(AccessController.java:559)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
	at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
	at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
	at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
	at java.nio.file.Files.readAttributes(Files.java:1686)
	at java.nio.file.Files.size(Files.java:2275)
	at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:549)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:509)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:433)
	at org.apache.lucene.analysis.TestLookaheadTokenFilter.testNeverCallingPeek(TestLookaheadTokenFilter.java:66)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.lucene.analysis.TestMockAnalyzer.testRandomRegexps

Error Message:
access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
	at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:9D624909112F86C9]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
	at java.security.AccessController.checkPermission(AccessController.java:559)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
	at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
	at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
	at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
	at java.nio.file.Files.readAttributes(Files.java:1686)
	at java.nio.file.Files.size(Files.java:2275)
	at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:549)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:509)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:428)
	at org.apache.lucene.analysis.TestMockAnalyzer.testRandomRegexps(TestMockAnalyzer.java:245)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.lucene.analysis.TestMockAnalyzer.testTwoChars

Error Message:
access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
	at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:983F3607E55D8D1F]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
	at java.security.AccessController.checkPermission(AccessController.java:559)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
	at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
	at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
	at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
	at java.nio.file.Files.readAttributes(Files.java:1686)
	at java.nio.file.Files.size(Files.java:2275)
	at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:549)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:509)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:428)
	at org.apache.lucene.analysis.TestMockAnalyzer.testTwoChars(TestMockAnalyzer.java:115)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.lucene.analysis.TestMockAnalyzer.testThreeChars

Error Message:
access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
	at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:CA5498F98659D9E]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
	at java.security.AccessController.checkPermission(AccessController.java:559)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
	at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
	at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
	at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
	at java.nio.file.Files.readAttributes(Files.java:1686)
	at java.nio.file.Files.size(Files.java:2275)
	at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:549)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:509)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:428)
	at org.apache.lucene.analysis.TestMockAnalyzer.testThreeChars(TestMockAnalyzer.java:136)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.lucene.analysis.TestMockAnalyzer.testUppercase

Error Message:
access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
	at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:9982189CA83F33D2]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
	at java.security.AccessController.checkPermission(AccessController.java:559)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
	at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
	at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
	at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
	at java.nio.file.Files.readAttributes(Files.java:1686)
	at java.nio.file.Files.size(Files.java:2275)
	at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:549)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:509)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:428)
	at org.apache.lucene.analysis.TestMockAnalyzer.testUppercase(TestMockAnalyzer.java:154)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.lucene.analysis.TestMockAnalyzer.testRandomStrings

Error Message:
access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
	at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:F63FA727BBC061EF]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
	at java.security.AccessController.checkPermission(AccessController.java:559)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
	at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
	at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
	at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
	at java.nio.file.Files.readAttributes(Files.java:1686)
	at java.nio.file.Files.size(Files.java:2275)
	at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:549)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:509)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:428)
	at org.apache.lucene.analysis.TestMockAnalyzer.testRandomStrings(TestMockAnalyzer.java:228)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.lucene.analysis.TestMockAnalyzer.testSingleChar

Error Message:
access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
	at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:F5153C673E9BD106]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
	at java.security.AccessController.checkPermission(AccessController.java:559)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
	at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
	at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
	at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
	at java.nio.file.Files.readAttributes(Files.java:1686)
	at java.nio.file.Files.size(Files.java:2275)
	at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:549)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:509)
	at org.apache.lucene.analysis.BaseTokenStreamTestCase.checkRandomData(BaseTokenStreamTestCase.java:428)
	at org.apache.lucene.analysis.TestMockAnalyzer.testSingleChar(TestMockAnalyzer.java:94)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.lucene.codecs.asserting.TestAssertingDocValuesFormat.testInvertedWrite

Error Message:
access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
	at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:67C2EA52669FBD06]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
	at java.security.AccessController.checkPermission(AccessController.java:559)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
	at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
	at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
	at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
	at java.nio.file.Files.readAttributes(Files.java:1686)
	at java.nio.file.Files.size(Files.java:2275)
	at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
	at org.apache.lucene.index.BasePostingsFormatTestCase.testInvertedWrite(BasePostingsFormatTestCase.java:1736)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.lucene.codecs.asserting.TestAssertingPostingsFormat.testInvertedWrite

Error Message:
access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
	at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:67C2EA52669FBD06]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
	at java.security.AccessController.checkPermission(AccessController.java:559)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
	at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
	at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
	at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
	at java.nio.file.Files.readAttributes(Files.java:1686)
	at java.nio.file.Files.size(Files.java:2275)
	at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
	at org.apache.lucene.index.BasePostingsFormatTestCase.testInvertedWrite(BasePostingsFormatTestCase.java:1736)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.lucene.codecs.lucene50.TestBlockPostingsFormat.testInvertedWrite

Error Message:
access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
	at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:67C2EA52669FBD06]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
	at java.security.AccessController.checkPermission(AccessController.java:559)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
	at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
	at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
	at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
	at java.nio.file.Files.readAttributes(Files.java:1686)
	at java.nio.file.Files.size(Files.java:2275)
	at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
	at org.apache.lucene.index.BasePostingsFormatTestCase.testInvertedWrite(BasePostingsFormatTestCase.java:1736)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.lucene.codecs.perfield.TestPerFieldPostingsFormat.testInvertedWrite

Error Message:
access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
	at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:67C2EA52669FBD06]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
	at java.security.AccessController.checkPermission(AccessController.java:559)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
	at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
	at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
	at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
	at java.nio.file.Files.readAttributes(Files.java:1686)
	at java.nio.file.Files.size(Files.java:2275)
	at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
	at org.apache.lucene.index.BasePostingsFormatTestCase.testInvertedWrite(BasePostingsFormatTestCase.java:1736)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.lucene.index.TestCustomNorms.testFloatNorms

Error Message:
access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
	at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:3796442599ECB78E]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
	at java.security.AccessController.checkPermission(AccessController.java:559)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
	at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
	at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
	at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
	at java.nio.file.Files.readAttributes(Files.java:1686)
	at java.nio.file.Files.size(Files.java:2275)
	at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
	at org.apache.lucene.index.TestCustomNorms.testFloatNorms(TestCustomNorms.java:52)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$7.evaluate(RandomizedRunner.java:863)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$8.evaluate(RandomizedRunner.java:877)
	at org.apache.lucene.util.TestRuleSetupTeardownChained$1.evaluate(TestRuleSetupTeardownChained.java:50)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at org.apache.lucene.util.TestRuleThreadAndTestName$1.evaluate(TestRuleThreadAndTestName.java:49)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl.forkTimeoutingTask(ThreadLeakControl.java:798)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$3.evaluate(ThreadLeakControl.java:458)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.runSingleTest(RandomizedRunner.java:836)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$3.evaluate(RandomizedRunner.java:738)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$4.evaluate(RandomizedRunner.java:772)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$5.evaluate(RandomizedRunner.java:783)
	at org.apache.lucene.util.AbstractBeforeAfterRule$1.evaluate(AbstractBeforeAfterRule.java:46)
	at org.apache.lucene.util.TestRuleStoreClassName$1.evaluate(TestRuleStoreClassName.java:42)
	at com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule$1.evaluate(SystemPropertiesInvariantRule.java:55)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.NoShadowingOrOverridesOnMethodsRule$1.evaluate(NoShadowingOrOverridesOnMethodsRule.java:39)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at org.apache.lucene.util.TestRuleAssertionsRequired$1.evaluate(TestRuleAssertionsRequired.java:54)
	at org.apache.lucene.util.TestRuleMarkFailure$1.evaluate(TestRuleMarkFailure.java:48)
	at org.apache.lucene.util.TestRuleIgnoreAfterMaxFailures$1.evaluate(TestRuleIgnoreAfterMaxFailures.java:65)
	at org.apache.lucene.util.TestRuleIgnoreTestSuites$1.evaluate(TestRuleIgnoreTestSuites.java:55)
	at com.carrotsearch.randomizedtesting.rules.StatementAdapter.evaluate(StatementAdapter.java:36)
	at com.carrotsearch.randomizedtesting.ThreadLeakControl$StatementRunner.run(ThreadLeakControl.java:365)
	at java.lang.Thread.run(Thread.java:745)


REGRESSION:  org.apache.lucene.index.TestDuelingCodecs.testEquals

Error Message:
access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")

Stack Trace:
java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
	at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:7AD9F402D6FD66D8]:0)
	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
	at java.security.AccessController.checkPermission(AccessController.java:559)
	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
	at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
	at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
	at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
	at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
	at java.nio.file.Files.readAttributes(Files.java:1686)
	at java.nio.file.Files.size(Files.java:2275)
	at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
	at org.apache.lucene.index.TestDuelingCodecs.createRandomIndex(TestDuelingCodecs.java:111)
	at org.apache.lucene.index.TestDuelingCodecs.testEquals(TestDuelingCodecs.java:148)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at com.carrotsearch.randomizedtesting.RandomizedRunner.invoke(RandomizedRunner.java:1618)
	at com.carrotsearch.randomizedtesting.RandomizedRunner$6.evaluate(RandomizedRunner.java:827)
	at com.carrotsearch.randomiz

[...truncated too long message...]

s.txt" "read")
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:67C2EA52669FBD06]:0)
   [junit4]    > 	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
   [junit4]    > 	at java.security.AccessController.checkPermission(AccessController.java:559)
   [junit4]    > 	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
   [junit4]    > 	at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
   [junit4]    > 	at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
   [junit4]    > 	at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
   [junit4]    > 	at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
   [junit4]    > 	at java.nio.file.Files.readAttributes(Files.java:1686)
   [junit4]    > 	at java.nio.file.Files.size(Files.java:2275)
   [junit4]    > 	at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
   [junit4]    > 	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
   [junit4]    > 	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:67)
   [junit4]    > 	at org.apache.lucene.index.BasePostingsFormatTestCase.testInvertedWrite(BasePostingsFormatTestCase.java:1736)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: leaving temporary files on disk at: /usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build/core/test/J0/temp/lucene.codecs.lucene50.TestBlockPostingsFormat 7EB6A79918C436DA-001
   [junit4]   2> NOTE: test params are: codec=CheapBastard, sim=DefaultSimilarity, locale=en_SG, timezone=Europe/Minsk
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=324570496,total=424673280
   [junit4]   2> NOTE: All tests run in this JVM: [TestManyFields, TestNorms, TestDocValuesTermsQuery, TestSearchWithThreads, TestCharsRef, TestScorerPerf, TestAssertingTermVectorsFormat, TestSortRescorer, TestDocIdSet, TestSumDocFreq, TestTermVectorsWriter, TestMatchAllDocsQuery, TestNRTThreads, TestOfflineSorter, TestPositionIncrement, TestCharTermAttributeImpl, TestDemo, InBeforeClass, InAfterClass, InTestMethod, NonStringProperties, TestTopFieldCollector, TestPayloadTermQuery, TestCachingTokenFilter, TestNRTCachingDirectory, TestForTooMuchCloning, TestSegmentTermEnum, TestTermsEnum, TestCachingCollector, TestNeedsScores, TestSubScorerFreqs, TestQueryRescorer, TestLongPostings, TestRollingBuffer, TestTryDelete, TestTermdocPerf, TestAutomatonQuery, TestVirtualMethod, Test4GBStoredFields, TestBooleanMinShouldMatch, TestReaderClosed, TestRegexpRandom, TestAssertingDocValuesFormat, TestAttributeSource, TestMathUtil, TestUTF32ToUTF8, TestIndexInput, TestPrefixFilter, TestScoreCachingWrappingScorer, TestMultiValuedNumericRangeQuery, TestSegmentMerger, TestInPlaceMergeSorter, TestRecyclingByteBlockAllocator, TestByteBlockPool, TestIndexWriterMerging, TestBytesStore, TestPackedInts, TestSearcherManager, TestDeletionPolicy, TestBooleanOr, TestBasics, TestFlushByRamOrCountsPolicy, TestSloppyPhraseQuery, TestIndexWriterMergePolicy, TestBoolean2, TestAtomicUpdate, TestIndexWriterCommit, TestNRTReaderWithThreads, TestTieredMergePolicy, TestDirectoryReader, TestCompressingStoredFieldsFormat, TestBlockPostingsFormat]
   [junit4] Completed on J0 in 18.52s, 18 tests, 1 error <<< FAILURES!

[...truncated 70 lines...]
   [junit4] Suite: org.apache.lucene.search.TestControlledRealTimeReopenThread
   [junit4] IGNOR/A 0.01s J2 | TestControlledRealTimeReopenThread.testCRTReopen
   [junit4]    > Assumption #1: 'awaitsfix' test group is disabled (@AwaitsFix(bugUrl=https://issues.apache.org/jira/browse/LUCENE-5737))
   [junit4]   2> NOTE: download the large Jenkins line-docs file by running 'ant get-jenkins-line-docs' in the lucene directory.
   [junit4]   2> NOTE: reproduce with: ant test  -Dtestcase=TestControlledRealTimeReopenThread -Dtests.method=testControlledRealTimeReopenThread -Dtests.seed=7EB6A79918C436DA -Dtests.multiplier=2 -Dtests.nightly=true -Dtests.slow=true -Dtests.linedocsfile=/home/jenkins/lucene-data/enwiki.random.lines.txt -Dtests.locale=pt_BR -Dtests.timezone=America/Grenada -Dtests.asserts=true -Dtests.file.encoding=US-ASCII
   [junit4] ERROR   0.01s J2 | TestControlledRealTimeReopenThread.testControlledRealTimeReopenThread <<<
   [junit4]    > Throwable #1: java.security.AccessControlException: access denied ("java.io.FilePermission" "/home/jenkins/lucene-data/enwiki.random.lines.txt" "read")
   [junit4]    > 	at __randomizedtesting.SeedInfo.seed([7EB6A79918C436DA:815FB65AEBC71ED6]:0)
   [junit4]    > 	at java.security.AccessControlContext.checkPermission(AccessControlContext.java:372)
   [junit4]    > 	at java.security.AccessController.checkPermission(AccessController.java:559)
   [junit4]    > 	at java.lang.SecurityManager.checkPermission(SecurityManager.java:549)
   [junit4]    > 	at java.lang.SecurityManager.checkRead(SecurityManager.java:888)
   [junit4]    > 	at sun.nio.fs.UnixPath.checkRead(UnixPath.java:795)
   [junit4]    > 	at sun.nio.fs.UnixFileAttributeViews$Basic.readAttributes(UnixFileAttributeViews.java:49)
   [junit4]    > 	at sun.nio.fs.UnixFileSystemProvider.readAttributes(UnixFileSystemProvider.java:144)
   [junit4]    > 	at java.nio.file.Files.readAttributes(Files.java:1686)
   [junit4]    > 	at java.nio.file.Files.size(Files.java:2275)
   [junit4]    > 	at org.apache.lucene.util.LineFileDocs.open(LineFileDocs.java:95)
   [junit4]    > 	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:63)
   [junit4]    > 	at org.apache.lucene.util.LineFileDocs.<init>(LineFileDocs.java:71)
   [junit4]    > 	at org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase.runTest(ThreadedIndexingAndSearchingTestCase.java:435)
   [junit4]    > 	at org.apache.lucene.search.TestControlledRealTimeReopenThread.testControlledRealTimeReopenThread(TestControlledRealTimeReopenThread.java:70)
   [junit4]    > 	at java.lang.Thread.run(Thread.java:745)
   [junit4]   2> NOTE: test params are: codec=Asserting(Lucene50): {test=TestBloomFilteredLucenePostings(BloomFilteringPostingsFormat(Lucene50(blocksize=128)))}, docValues:{}, sim=RandomSimilarityProvider(queryNorm=true,coord=yes): {test=IB SPL-LZ(0.3)}, locale=pt_BR, timezone=America/Grenada
   [junit4]   2> NOTE: FreeBSD 9.1-RELEASE-p3 amd64/Oracle Corporation 1.7.0_65 (64-bit)/cpus=16,threads=1,free=176860360,total=434634752
   [junit4]   2> NOTE: All tests run in this JVM: [TestTerms, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, Nested, TestSloppyMath, TestDuelingCodecs, TestSimilarityProvider, TestMixedDocValuesUpdates, TestDateSort, TestSearchForDuplicates, TestPrefixInBooleanQuery, TestSortRandom, TestPrefixCodedTerms, TestMultiFields, TestSimilarity2, TestDirectory, Test2BPostings, TestSizeBoundedForceMerge, TestPrefixRandom, TestMinimize, Nested2, Nested, TestFilterLeafReader, TestSpansAdvanced2, TestDeterminism, TestMergeRateLimiter, TestExceedMaxTermLength, TestCodecs, TestIndexWriterLockRelease, TestOmitTf, TestIndexWriterOutOfFileDescriptors, TestToken, TestLock, TestSearchAfter, TestCustomNorms, TestComplexExplanations, TestTermVectors, TestBlockPostingsFormat3, TestTopDocsMerge, TestFileSwitchDirectory, TestFilterCachingPolicy, TestMockCharFilter, TestFilterIterator, TestTermsEnum2, TestRamUsageEstimator, TestCrash, TestParallelTermEnum, Test2BTerms, TestDocInverterPerFieldErrorInfo, TestStressNRT, TestParallelCompositeReader, TestIndexWriterOutOfMemory, TestMinShouldMatch2, TestSegmentReader, TestFastDecompressionMode, TestIndexWriterNRTIsCurrent, TestFieldCacheRewriteMethod, TestFieldsReader, TestCustomSearcherSort, TestDocValuesIndexing, TestNumericUtils, TestDeterminizeLexicon, TestIndexWriterOnDiskFull, TestSpans, TestMixedCodecs, TestTransactionRollback, TestFastCompressionMode, TestLazyProxSkipping, TestRamUsageEstimatorOnWildAnimals, TestMergeSchedulerExternal, TestThreadedForceMerge, TestSloppyPhraseQuery2, TestCodecUtil, TestDocValues, TestExitableDirectoryReader, TestFieldReuse, TestIndexReaderClose, TestIndexWriterThreadsToSegments, TestInfoStream, TestLegacyPostings, TestLogMergePolicy, TestNRTReaderCleanup, TestMockFilesystems, TestControlledRealTimeReopenThread]
   [junit4] Completed on J2 in 0.07s, 5 tests, 1 error, 1 skipped <<< FAILURES!

[...truncated 162 lines...]
BUILD FAILED
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:536: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:477: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/build.xml:61: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/extra-targets.xml:39: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/build.xml:49: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:1370: The following error occurred while executing this line:
/usr/home/jenkins/jenkins-slave/workspace/Lucene-Solr-NightlyTests-5.x/lucene/common-build.xml:976: There were test failures: 418 suites, 3307 tests, 2 suite-level errors, 33 errors, 38 ignored (28 assumptions)

Total time: 31 minutes 40 seconds
Build step 'Invoke Ant' marked build as failure
Archiving artifacts
Sending artifact delta relative to Lucene-Solr-NightlyTests-5.x #716
Archived 5 artifacts
Archive block size is 32768
Received 0 blocks and 4070516 bytes
Compression is 0.0%
Took 1.1 sec
Recording test results
Email was triggered for: Failure
Sending email for trigger: Failure